pax_global_header00006660000000000000000000000064132427305770014524gustar00rootroot0000000000000052 comment=e4c2b379ee94c7c7cd16ff0f264b508e508ff4a1 diskimage-builder-2.11.0/000077500000000000000000000000001324273057700151665ustar00rootroot00000000000000diskimage-builder-2.11.0/.gitignore000066400000000000000000000003251324273057700171560ustar00rootroot00000000000000.coverage coverage.xml cover/* *~ .testrepository *.sw? #*# *.pyc .tox *.egg *.egg-info dist *.qcow2 *.raw *.initrd *.vmlinuz /*-manifests /*.d !/.zuul.d build AUTHORS ChangeLog bin/diskimage_builder *.bak *.orig diskimage-builder-2.11.0/.gitreview000066400000000000000000000001261324273057700171730ustar00rootroot00000000000000[gerrit] host=review.openstack.org port=29418 project=openstack/diskimage-builder.git diskimage-builder-2.11.0/.testr.conf000066400000000000000000000006241324273057700172560ustar00rootroot00000000000000[DEFAULT] test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ OS_LOG_CAPTURE=${OS_LOG_CAPTURE:-1} \ OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ OS_DEBUG=${OS_DEBUG:-0} \ ${PYTHON:-python} -m subunit.run discover . $LISTOPT $IDOPTION test_id_option=--load-list $IDFILE test_list_option=--list diskimage-builder-2.11.0/.zuul.d/000077500000000000000000000000001324273057700164655ustar00rootroot00000000000000diskimage-builder-2.11.0/.zuul.d/jobs.yaml000066400000000000000000000132351324273057700203120ustar00rootroot00000000000000- project: templates: - tripleo-buildimage-jobs check: jobs: - dib-dsvm-functests-python2-ubuntu-trusty: branches: (master|feature/v2) irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - dib-dsvm-functests-python2-ubuntu-trusty-extras: voting: false branches: (master|feature/v2) irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - dib-dsvm-functests-python2-ubuntu-trusty-image: voting: false branches: (master|feature/v2) irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - dib-dsvm-functests-python3: irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - dib-dsvm-functests-python3-extras: voting: false irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - dib-dsvm-functests-python3-image: voting: false irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - dib-dsvm-functests-python2-centos-7: irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - dib-dsvm-functests-python2-centos-7-extras: voting: false irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - dib-dsvm-functests-python2-centos-7-image: voting: false irrelevant-files: - ^.*\.rst$ - ^doc/.*$ gate: jobs: - dib-dsvm-functests-python2-ubuntu-trusty: branches: (master|feature/v2) irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - dib-dsvm-functests-python3: irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - dib-dsvm-functests-python2-centos-7: irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - tripleo-buildimage-overcloud-full-centos-7 - job: name: dib-dsvm-functests-python2-centos-7 parent: legacy-dsvm-base run: playbooks/legacy/dib-dsvm-functests-python2-centos-7/run.yaml post-run: playbooks/legacy/dib-dsvm-functests-python2-centos-7/post.yaml timeout: 3600 nodeset: legacy-centos-7 required-projects: - openstack/diskimage-builder - job: name: dib-dsvm-functests-python2-centos-7-extras parent: legacy-dsvm-base run: playbooks/legacy/dib-dsvm-functests-python2-centos-7-extras/run.yaml post-run: playbooks/legacy/dib-dsvm-functests-python2-centos-7-extras/post.yaml timeout: 3600 nodeset: legacy-centos-7 required-projects: - openstack/diskimage-builder - job: name: dib-dsvm-functests-python2-centos-7-image parent: legacy-dsvm-base run: playbooks/legacy/dib-dsvm-functests-python2-centos-7-image/run.yaml post-run: playbooks/legacy/dib-dsvm-functests-python2-centos-7-image/post.yaml timeout: 3600 nodeset: legacy-centos-7 required-projects: - openstack/diskimage-builder - job: name: dib-dsvm-functests-python2-opensuse-423 parent: legacy-dsvm-base run: playbooks/legacy/dib-dsvm-functests-python2-opensuse-423/run.yaml post-run: playbooks/legacy/dib-dsvm-functests-python2-opensuse-423/post.yaml timeout: 3600 nodeset: legacy-opensuse-423 required-projects: - openstack/diskimage-builder - job: name: dib-dsvm-functests-python2-ubuntu-trusty parent: legacy-dsvm-base run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty/run.yaml post-run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty/post.yaml timeout: 3600 nodeset: legacy-ubuntu-trusty required-projects: - openstack/diskimage-builder - job: name: dib-dsvm-functests-python2-ubuntu-trusty-extras parent: legacy-dsvm-base run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-extras/run.yaml post-run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-extras/post.yaml timeout: 3600 nodeset: legacy-ubuntu-trusty required-projects: - openstack/diskimage-builder - job: name: dib-dsvm-functests-python2-ubuntu-trusty-image parent: legacy-dsvm-base run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-image/run.yaml post-run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-image/post.yaml timeout: 3600 nodeset: legacy-ubuntu-trusty required-projects: - openstack/diskimage-builder - job: name: dib-dsvm-functests-python3 parent: legacy-dsvm-base run: playbooks/legacy/dib-dsvm-functests-python3/run.yaml post-run: playbooks/legacy/dib-dsvm-functests-python3/post.yaml timeout: 3600 required-projects: - openstack/diskimage-builder - job: name: dib-dsvm-functests-python3-extras parent: legacy-dsvm-base run: playbooks/legacy/dib-dsvm-functests-python3-extras/run.yaml post-run: playbooks/legacy/dib-dsvm-functests-python3-extras/post.yaml timeout: 3600 required-projects: - openstack/diskimage-builder - job: name: dib-dsvm-functests-python3-image parent: legacy-dsvm-base run: playbooks/legacy/dib-dsvm-functests-python3-image/run.yaml post-run: playbooks/legacy/dib-dsvm-functests-python3-image/post.yaml timeout: 3600 required-projects: - openstack/diskimage-builder - job: name: dib-dsvm-functests-python3-opensuse-423 parent: legacy-dsvm-base run: playbooks/legacy/dib-dsvm-functests-python3-opensuse-423/run.yaml post-run: playbooks/legacy/dib-dsvm-functests-python3-opensuse-423/post.yaml timeout: 3600 nodeset: legacy-opensuse-423 required-projects: - openstack/diskimage-builder diskimage-builder-2.11.0/LICENSE000066400000000000000000000261361324273057700162030ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diskimage-builder-2.11.0/README.rst000066400000000000000000000036211324273057700166570ustar00rootroot00000000000000Image building tools for OpenStack ================================== ``diskimage-builder`` is a flexible suite of components for building a wide-range of disk images, filesystem images and ramdisk images for use with OpenStack. This repository has the core functionality for building such images, both virtual and bare metal. Images are composed using `elements`; while fundamental elements are provided here, individual projects have the flexibility to customise the image build with their own elements. For example:: $ DIB_RELEASE=trusty disk-image-create -o ubuntu-trusty.qcow2 vm ubuntu will create a bootable Ubuntu Trusty based ``qcow2`` image. ``diskimage-builder`` is useful to anyone looking to produce customised images for deployment into clouds. These tools are the components of `TripleO `__ that are responsible for building disk images. They are also used extensively to build images for testing OpenStack itself, particularly with `nodepool `__. Platforms supported include Ubuntu, CentOS, RHEL and Fedora. Full documentation, the source of which is in ``doc/source/``, is published at: * https://docs.openstack.org/diskimage-builder/latest/ Copyright ========= Copyright 2012 Hewlett-Packard Development Company, L.P. Copyright (c) 2012 NTT DOCOMO, INC. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diskimage-builder-2.11.0/babel.cfg000066400000000000000000000000201324273057700167040ustar00rootroot00000000000000[python: **.py] diskimage-builder-2.11.0/bin/000077500000000000000000000000001324273057700157365ustar00rootroot00000000000000diskimage-builder-2.11.0/bin/dib-lint000077500000000000000000000227331324273057700173750ustar00rootroot00000000000000#!/bin/bash # Copyright 2014 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. # This script checks all files in the "elements" directory for some # common mistakes and exits with a non-zero status if it finds any. set -eu set -o pipefail ELEMENTS_DIR=${ELEMENTS_DIR:-diskimage_builder/elements} LIB_DIR=${LIB_DIR:-diskimage_builder/lib} parse_exclusions() { # Per-file exclusions # Example: # dib-lint: disable=sete setpipefail local filename=$1 local disable_pattern="# dib-lint: disable=" local exclusions=$(grep "^$disable_pattern.*$" $filename | sed "s/$disable_pattern//g") # Global exclusions read from tox.ini # Example section in tox.ini: # [dib-lint] # ignore = sete setu section="dib-lint" option="ignore" global_exclusions=$(python - < ${UNSORTED} sort ${UNSORTED} > ${SORTED} if [ -n "$(diff -c ${UNSORTED} ${SORTED})" ]; then error "$i is not sorted alphabetically" diff -y ${UNSORTED} ${SORTED} fi fi # for consistency, let's just use #!/bin/bash everywhere (not # /usr/bin/env, etc) regex='^#!.*bash' if [[ "$firstline" =~ $regex && "$firstline" != "#!/bin/bash" ]]; then error "$i : only use #!/bin/bash for scripts" fi # Check that all scripts are set -eu -o pipefail and look for # DIB_DEBUG_TRACE # NOTE(bnemec): This doesn't verify that the set call occurs high # enough in the file to be useful, but hopefully nobody will be # sticking set calls at the end of their file to trick us. And if # they are, that's easy enough to catch in reviews. # Also, this is only going to check bash scripts - we've decided to # explicitly require bash for any scripts that don't have a specific # need to run under other shells, and any exceptions to that rule # may not want these checks either. if [[ "$firstline" =~ '#!/bin/bash' ]]; then if ! excluded sete; then if [ -z "$(grep "^set -[^ ]*e" $i)" ]; then error "$i is not set -e" fi fi if ! excluded setu; then if [ -z "$(grep "^set -[^ ]*u" $i)" ]; then error "$i is not set -u" fi fi if ! excluded setpipefail; then if [ -z "$(grep "^set -o pipefail" $i)" ]; then error "$i is not set -o pipefail" fi fi if ! excluded dibdebugtrace; then if [ -z "$(grep "DIB_DEBUG_TRACE" $i)" ]; then error "$i does not follow DIB_DEBUG_TRACE" fi fi fi # check that environment files don't "set -x" and they have no executable # bits set if [[ "$i" =~ (environment.d) ]]; then if grep -q "set -x" $i; then error "Environment file $i should not set tracing" fi if [[ -x $i ]]; then error "Environment file $i should not be marked as executable" fi fi # check for # export FOO=$(bar) # calls. These are dangerous, because the export hides the return # code of the $(bar) call. Split this into 2 lines and -e will # fail on the assignment if grep -q 'export .*\$(' $i; then error "Split export and assignments in $i" fi # check that sudo calls in phases run outside the chroot look # "safe"; meaning that they seem to operate within the chroot # somehow. This is not fool-proof, but catches egregious errors, # and makes you think about it if you're doing something outside # the box. if ! excluded safe_sudo; then if [[ $(dirname $i) =~ (root.d|extra-data.d|block-device.d|finalise.d|cleanup.d) ]]; then while read LINE do if [[ $LINE =~ "sudo " ]]; then # messy regex ahead! Don't match: # - explicitly ignored # - basic comments # - install-packages ... sudo ... # - any of the paths passed into the out-of-chroot elements if [[ $LINE =~ (dib-lint: safe_sudo|^#|install-packages|TARGET_ROOT|IMAGE_BLOCK_DEVICE|TMP_MOUNT_PATH|TMP_IMAGE_PATH) ]]; then continue fi error "$i : potentially unsafe sudo\n -- $LINE" fi done < $i fi fi # check that which calls are not used. It is not built in and is missing # from some constrained environments if ! excluded which; then while read LINE do if [[ $LINE =~ "which " ]]; then # Don't match: # - explicitly ignored # - commented if [[ $LINE =~ (dib-lint: which|^#) ]]; then continue fi error "$i : potential use of which\n -- $LINE" fi done < $i fi done echo "Checking indents..." for i in $(find $ELEMENTS_DIR -type f -and -name '*.rst' -or -type f -executable) \ $(find $LIB_DIR -type f); do # Skip files in .gitignore if git check-ignore -q "$i" ; then echo Skipping $i continue fi # Check for tab indentation if ! excluded tabindent; then if grep -q $'^ *\t' ${i}; then error "$i contains tab characters" fi fi if ! excluded newline; then if [ "$(tail -c 1 $i)" != "" ]; then error "No newline at end of file: $i" fi fi done if ! excluded mddocs; then md_docs=$(find $ELEMENTS_DIR -name '*.md') if [ -n "$md_docs" ]; then error ".md docs found: $md_docs" fi fi echo "Checking YAML parsing..." for i in $(find $ELEMENTS_DIR -type f -name '*.yaml'); do echo "Parsing $i" py_check=" import yaml import sys try: objs = yaml.safe_load(open('$i')) except yaml.parser.ParserError: sys.exit(1) " if ! python -c "$py_check"; then error "$i is not a valid YAML file" fi done echo "Checking pkg-map files..." for i in $(find $ELEMENTS_DIR -type f \ -name 'pkg-map' -a \! -executable); do echo "Parsing $i" py_check=" import json import sys try: objs = json.load(open('$i')) except ValueError: sys.exit(1) " if ! python -c "$py_check"; then error "$i is not a valid JSON file" fi done if [[ $rc == 0 ]]; then echo "PASS" else echo "*** FAIL: Some tests failed!" fi exit $rc diskimage-builder-2.11.0/bindep.txt000066400000000000000000000005071324273057700171720ustar00rootroot00000000000000# This is a cross-platform list tracking distribution packages needed by tests; # see https://docs.openstack.org/infra/bindep/ for additional information. squashfs-tools [!platform:suse] squashfs [platform:suse] zypper [!platform:redhat !platform:ubuntu-trusty] gnupg2 [!platform:redhat !platform:ubuntu-trusty !platform:suse] diskimage-builder-2.11.0/contrib/000077500000000000000000000000001324273057700166265ustar00rootroot00000000000000diskimage-builder-2.11.0/contrib/setup-gate-mirrors.sh000077500000000000000000000035701324273057700227430ustar00rootroot00000000000000#!/bin/bash set -x # # This tool creates repo/sources files that point to the mirrors for # the host region in the OpenStack CI gate. # # This pre-created on CI nodes by slave scripts source /etc/ci/mirror_info.sh # Tests should probe for this directory and then use the repos/sources # files inside it for the gate tests. BASE_DIR=$WORKSPACE/dib-mirror mkdir -p $BASE_DIR ## REPOS # all should start with "dib-mirror-" # gpg check turned off, because we don't have the keys outside the chroot # fedora-minimal FEDORA_MIN_DIR=$BASE_DIR/fedora-minimal/yum.repos.d mkdir -p $FEDORA_MIN_DIR cat < $FEDORA_MIN_DIR/dib-mirror-fedora.repo [fedora] name=Fedora \$releasever - \$basearch failovermethod=priority baseurl=$NODEPOOL_FEDORA_MIRROR/releases/\$releasever/Everything/\$basearch/os/ enabled=1 metadata_expire=7d gpgcheck=0 skip_if_unavailable=False deltarpm=False deltarpm_percentage=0 EOF cat < $FEDORA_MIN_DIR/dib-mirror-fedora-updates.repo [updates] name=Fedora \$releasever - \$basearch - Updates failovermethod=priority baseurl=$NODEPOOL_FEDORA_MIRROR/updates/\$releasever/\$basearch/ enabled=1 gpgcheck=0 metadata_expire=6h skip_if_unavailable=False deltarpm=False deltarpm_percentage=0 EOF # Centos Minimal CENTOS_MIN_DIR=$BASE_DIR/centos-minimal/yum.repos.d mkdir -p $CENTOS_MIN_DIR cat < $CENTOS_MIN_DIR/dib-mirror-base.repo [base] name=CentOS-\$releasever - Base baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/os/\$basearch/ gpgcheck=0 EOF cat < $CENTOS_MIN_DIR/dib-mirror-updates.repo #released updates [updates] name=CentOS-\$releasever - Updates baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/updates/\$basearch/ gpgcheck=0 EOF cat < $CENTOS_MIN_DIR/dib-mirror-extras.repo #additional packages that may be useful [extras] name=CentOS-\$releasever - Extras baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/extras/\$basearch/ gpgcheck=0 EOF ## apt sources (todo) diskimage-builder-2.11.0/diskimage_builder/000077500000000000000000000000001324273057700206315ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/__init__.py000066400000000000000000000000001324273057700227300ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/000077500000000000000000000000001324273057700232425ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/__init__.py000066400000000000000000000000001324273057700253410ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/blockdevice.py000066400000000000000000000426241324273057700260760ustar00rootroot00000000000000# Copyright 2016-2017 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import codecs import collections import json import logging import os import pickle import pprint import shutil import yaml from diskimage_builder.block_device.config import config_tree_to_graph from diskimage_builder.block_device.config import create_graph from diskimage_builder.block_device.exception import \ BlockDeviceSetupException from diskimage_builder.block_device.utils import exec_sudo logger = logging.getLogger(__name__) def _load_json(file_name): """Load file from .json file on disk, return None if not existing""" if os.path.exists(file_name): with codecs.open(file_name, encoding="utf-8", mode="r") as fd: return json.load(fd) return None class BlockDeviceState(collections.MutableMapping): """The global state singleton An reference to an instance of this object is saved into nodes as a global repository. It wraps a single dictionary "state" and provides a few helper functions. The state ends up used in two contexts: - The node list (including this state) is pickled and dumped between cmd_create() and later cmd_* calls that need to call the nodes. - Some other cmd_* calls, such as cmd_writefstab, only need access to values inside the state and not the whole node list, and load it from the json dump created after cmd_create() """ # XXX: # - we could implement getters/setters such that if loaded from # disk, the state is read-only? or make it append-only # (i.e. you can't overwrite existing keys) def __init__(self, filename=None): """Initialise state :param filename: if :param:`filename` is passed and exists, it will be loaded as the state. If it does not exist an exception is raised. If :param:`filename` is not passed, state will be initalised to a blank dictionary. """ if filename: if not os.path.exists(filename): raise BlockDeviceSetupException("State dump not found") else: self.state = _load_json(filename) assert self.state is not None else: self.state = {} def __delitem__(self, key): del self.state[key] def __getitem__(self, key): return self.state[key] def __setitem__(self, key, value): self.state[key] = value def __iter__(self): return iter(self.state) def __len__(self): return len(self.state) def save_state(self, filename): """Persist the state to disk :param filename: The file to persist state to """ logger.debug("Writing state to: %s", filename) self.debug_dump() with open(filename, "w") as fd: json.dump(self.state, fd) def debug_dump(self): """Log state to debug""" # This is pretty good for human consumption, but maybe a bit # verbose. nice_output = pprint.pformat(self.state, width=40) for l in nice_output.split('\n'): logger.debug('{0:{fill}{align}50}'.format(l, fill=' ', align='<')) class BlockDevice(object): """Handles block devices. This class handles the complete setup and deletion of all aspects of the block device level. A typical call sequence: cmd_init: initialize the block device level config. After this call it is possible to e.g. query information from the (partially automatic generated) internal state like root-label. cmd_getval: retrieve information about the (internal) block device state like the block image device (for bootloader) or the root-label (for writing fstab). cmd_create: creates all the different aspects of the block device. When this call is successful, the complete block level device is set up, filesystems are created and are mounted at the correct position. After this call it is possible to copy / install all the needed files into the appropriate directories. cmd_writefstab: creates the (complete) fstab for the system. cmd_umount: unmount and detaches all directories and used many resources. After this call the used (e.g.) images are still available for further handling, e.g. converting from raw in some other format. cmd_cleanup: removes everything that was created with the 'cmd_create' call, i.e. all images files themselves and internal temporary configuration. cmd_delete: unmounts and removes everything that was created during the 'cmd_create' all. This call should be used in error conditions when there is the need to remove all allocated resources immediately and as good as possible. From the functional point of view this is mostly the same as a call to 'cmd_umount' and 'cmd_cleanup' - but is typically more error tolerance. In a script this should be called in the following way: dib-block-device init ... # From that point the database can be queried, like ROOT_LABEL=$(dib-block-device getval root-label) Please note that currently the dib-block-device executable can only be used outside the chroot. dib-block-device create ... trap "dib-block-device delete ..." EXIT # copy / install files dib-block-device umount ... # convert image(s) dib-block-device cleanup ... trap - EXIT """ def _merge_rootfs_params(self): """Merge rootfs related parameters into configuration To maintain compatability with some old block-device environment variables from before we had a specific block-device config, disk-image-create provides a "parameters" file that translates the old bash-environment variables into a YAML based configuration file (``self.params``). Here we merge the values in this parameters file that relate to the root file-system into the final configuration. We look for the ``mkfs_root`` node in the new config, and pull the relevant settings from the parameters into it. """ for entry in self.config: for k, v in entry.items(): if k == 'mkfs': if 'name' not in v: continue if v['name'] != 'mkfs_root': continue if 'type' not in v \ and 'root-fs-type' in self.params: v['type'] = self.params['root-fs-type'] if 'opts' not in v \ and 'root-fs-opts' in self.params: v['opts'] = self.params['root-fs-opts'] if 'label' not in v \ and 'root-label' in self.params: if self.params['root-label'] is not None: v['label'] = self.params['root-label'] else: # The default label is "cloudimg-rootfs" # for historical reasons (upstream # images/EC2 defaults/cloud-init etc). We # want to remain backwards compatible, but # unfortunately that's too long for XFS so # we've decided on 'img-rootfs' in that # case. Note there's separate checks if # the label is specified explicitly. if v.get('type') == 'xfs': v['label'] = 'img-rootfs' else: v['label'] = 'cloudimg-rootfs' def __init__(self, params): """Create BlockDevice object Arguments: :param params: YAML file from --params """ logger.debug("Creating BlockDevice object") self.params = params logger.debug("Params [%s]", self.params) self.state_dir = os.path.join( self.params['build-dir'], "states/block-device") self.state_json_file_name \ = os.path.join(self.state_dir, "state.json") self.config_json_file_name \ = os.path.join(self.state_dir, "config.json") self.node_pickle_file_name \ = os.path.join(self.state_dir, "nodes.pickle") self.config = _load_json(self.config_json_file_name) # This needs to exists for the state and config files try: os.makedirs(self.state_dir) except OSError: pass def cmd_init(self): """Initialize block device setup This initializes the block device setup layer. One major task is to parse and check the configuration, write it down for later examiniation and execution. """ with open(self.params['config'], "rt") as config_fd: self.config = yaml.safe_load(config_fd) logger.debug("Config before merge [%s]", self.config) self.config = config_tree_to_graph(self.config) logger.debug("Config before merge [%s]", self.config) self._merge_rootfs_params() logger.debug("Final config [%s]", self.config) # Write the final config with open(self.config_json_file_name, "wt") as fd: json.dump(self.config, fd) logger.info("Wrote final block device config to [%s]", self.config_json_file_name) def _config_get_mount(self, path): for entry in self.config: for k, v in entry.items(): if k == 'mount' and v['mount_point'] == path: return v assert False def _config_get_all_mount_points(self): rvec = [] for entry in self.config: for k, v in entry.items(): if k == 'mount': rvec.append(v['mount_point']) return rvec def _config_get_mkfs(self, name): for entry in self.config: for k, v in entry.items(): if k == 'mkfs' and v['name'] == name: return v assert False def cmd_getval(self, symbol): """Retrieve value from block device level The value of SYMBOL is printed to stdout. This is intended to be captured into bash-variables for backward compatibility (non python) access to internal configuration. Arguments: :param symbol: the symbol to get """ logger.info("Getting value for [%s]", symbol) if symbol == "root-label": root_mount = self._config_get_mount("/") root_fs = self._config_get_mkfs(root_mount['base']) logger.debug("root-label [%s]", root_fs['label']) print("%s" % root_fs['label']) return 0 if symbol == "root-fstype": root_mount = self._config_get_mount("/") root_fs = self._config_get_mkfs(root_mount['base']) logger.debug("root-fstype [%s]", root_fs['type']) print("%s" % root_fs['type']) return 0 if symbol == 'mount-points': mount_points = self._config_get_all_mount_points() # we return the mountpoints joined by a pipe, because it is not # a valid char in directories, so it is a safe separator for the # mountpoints list print("%s" % "|".join(mount_points)) return 0 # the following symbols all come from the global state # dictionary. They can only be accessed after the state has # been dumped; i.e. after cmd_create() called. state = BlockDeviceState(self.state_json_file_name) # The path to the .raw file for conversion if symbol == 'image-path': print("%s" % state['blockdev']['image0']['image']) return 0 # This is the loopback device where the above image is setup if symbol == 'image-block-device': print("%s" % state['blockdev']['image0']['device']) return 0 # Full list of created devices by name. Some bootloaders, for # example, want to be able to see their boot partitions to # copy things in. Intended to be read into a bash array if symbol == 'image-block-devices': out = "" for k, v in state['blockdev'].items(): out += " [%s]=%s " % (k, v['device']) print(out) return 0 logger.error("Invalid symbol [%s] for getval", symbol) return 1 def cmd_writefstab(self): """Creates the fstab""" logger.info("Creating fstab") # State should have been created by prior calls; we only need # the dict state = BlockDeviceState(self.state_json_file_name) tmp_fstab = os.path.join(self.state_dir, "fstab") with open(tmp_fstab, "wt") as fstab_fd: # This gives the order in which this must be mounted for mp in state['mount_order']: logger.debug("Writing fstab entry for [%s]", mp) fs_base = state['mount'][mp]['base'] fs_name = state['mount'][mp]['name'] fs_val = state['filesys'][fs_base] if 'label' in fs_val: diskid = "LABEL=%s" % fs_val['label'] else: diskid = "UUID=%s" % fs_val['uuid'] # If there is no fstab entry - do not write anything if 'fstab' not in state: continue if fs_name not in state['fstab']: continue options = state['fstab'][fs_name]['options'] dump_freq = state['fstab'][fs_name]['dump-freq'] fsck_passno = state['fstab'][fs_name]['fsck-passno'] fstab_fd.write("%s %s %s %s %s %s\n" % (diskid, mp, fs_val['fstype'], options, dump_freq, fsck_passno)) target_etc_dir = os.path.join(self.params['build-dir'], 'built', 'etc') exec_sudo(['mkdir', '-p', target_etc_dir]) exec_sudo(['cp', tmp_fstab, os.path.join(target_etc_dir, "fstab")]) return 0 def cmd_create(self): """Creates the block device""" logger.info("create() called") logger.debug("Using config [%s]", self.config) # Create a new, empty state state = BlockDeviceState() try: dg, call_order = create_graph(self.config, self.params, state) for node in call_order: node.create() except Exception: logger.exception("Create failed; rollback initiated") reverse_order = reversed(call_order) for node in reverse_order: node.rollback() # save the state for debugging state.save_state(self.state_json_file_name) logger.error("Rollback complete, exiting") raise # dump state and nodes, in order # XXX: we only dump the call_order (i.e. nodes) not the whole # graph here, because later calls do not need the graph # at this stage. might they? state.save_state(self.state_json_file_name) pickle.dump(call_order, open(self.node_pickle_file_name, 'wb')) logger.info("create() finished") return 0 def cmd_umount(self): """Unmounts the blockdevice and cleanup resources""" # If the state is not here, cmd_cleanup removed it? Nothing # more to do? # XXX: better understand this... if not os.path.exists(self.node_pickle_file_name): logger.info("State already cleaned - no way to do anything here") return 0 call_order = pickle.load(open(self.node_pickle_file_name, 'rb')) reverse_order = reversed(call_order) for node in reverse_order: node.umount() return 0 def cmd_cleanup(self): """Cleanup all remaining relicts - in good case""" # Cleanup must be done in reverse order try: call_order = pickle.load(open(self.node_pickle_file_name, 'rb')) except IOError: raise BlockDeviceSetupException("Pickle file not found") reverse_order = reversed(call_order) for node in reverse_order: node.cleanup() logger.info("Removing temporary state dir [%s]", self.state_dir) shutil.rmtree(self.state_dir) return 0 def cmd_delete(self): """Cleanup all remaining relicts - in case of an error""" # Deleting must be done in reverse order try: call_order = pickle.load(open(self.node_pickle_file_name, 'rb')) except IOError: raise BlockDeviceSetupException("Pickle file not found") reverse_order = reversed(call_order) for node in reverse_order: node.delete() logger.info("Removing temporary state dir [%s]", self.state_dir) shutil.rmtree(self.state_dir) return 0 diskimage-builder-2.11.0/diskimage_builder/block_device/cmd.py000066400000000000000000000102041324273057700243540ustar00rootroot00000000000000# Copyright 2016-2017 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import argparse import logging import os import sys import yaml from diskimage_builder.block_device.blockdevice import BlockDevice from diskimage_builder import logging_config logger = logging.getLogger(__name__) class BlockDeviceCmd(object): def cmd_init(self): self.bd.cmd_init() def cmd_getval(self): self.bd.cmd_getval(self.args.symbol) def cmd_create(self): self.bd.cmd_create() def cmd_umount(self): self.bd.cmd_umount() def cmd_cleanup(self): self.bd.cmd_cleanup() def cmd_delete(self): self.bd.cmd_delete() def cmd_writefstab(self): self.bd.cmd_writefstab() def main(self): logging_config.setup() parser = argparse.ArgumentParser(description="DIB Block Device helper") parser.add_argument('--params', required=False, help="YAML file containing parameters for" "block-device handling. Default is " "DIB_BLOCK_DEVICE_PARAMS_YAML") subparsers = parser.add_subparsers(title='commands', description='valid commands', dest='command', help='additional help') cmd_init = subparsers.add_parser('init', help='Initialize configuration') cmd_init.set_defaults(func=self.cmd_init) cmd_getval = subparsers.add_parser('getval', help='Retrieve information about' 'internal state') cmd_getval.set_defaults(func=self.cmd_getval) cmd_getval.add_argument('symbol', help='symbol to print') cmd_create = subparsers.add_parser('create', help='Create the block device') cmd_create.set_defaults(func=self.cmd_create) cmd_umount = subparsers.add_parser('umount', help='Unmount blockdevice and' 'cleanup resources') cmd_umount.set_defaults(func=self.cmd_umount) cmd_cleanup = subparsers.add_parser('cleanup', help='Final cleanup') cmd_cleanup.set_defaults(func=self.cmd_cleanup) cmd_delete = subparsers.add_parser('delete', help='Error cleanup') cmd_delete.set_defaults(func=self.cmd_delete) cmd_writefstab = subparsers.add_parser('writefstab', help='Create fstab for system') cmd_writefstab.set_defaults(func=self.cmd_writefstab) self.args = parser.parse_args() # Find, open and parse the parameters file if not self.args.params: if 'DIB_BLOCK_DEVICE_PARAMS_YAML' in os.environ: param_file = os.environ['DIB_BLOCK_DEVICE_PARAMS_YAML'] else: parser.error( "DIB_BLOCK_DEVICE_PARAMS_YAML or --params not set") else: param_file = self.args.params logger.info("params [%s]", param_file) try: with open(param_file) as f: self.params = yaml.safe_load(f) except Exception: logger.exception("Failed to open parameter YAML") sys.exit(1) # Setup main BlockDevice object from args self.bd = BlockDevice(self.params) self.args.func() def main(): bdc = BlockDeviceCmd() return bdc.main() if __name__ == "__main__": sys.exit(main()) diskimage-builder-2.11.0/diskimage_builder/block_device/config.py000066400000000000000000000220341324273057700250620ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import logging import networkx as nx import os from stevedore import extension from diskimage_builder.block_device.exception import \ BlockDeviceSetupException from diskimage_builder.block_device.plugin import NodeBase from diskimage_builder.block_device.plugin import PluginBase logger = logging.getLogger(__name__) _extensions = extension.ExtensionManager( namespace='diskimage_builder.block_device.plugin', invoke_on_load=False) # check if a given name is registered as a plugin def is_a_plugin(name): return any( _extensions.map(lambda x: x.name == name)) def recurse_config(config, parent_base=None): """Convert a config "tree" to it's canonical name/base graph version This is a recursive function to convert a YAML layout "tree" config into a "flat" graph-based config. Arguments: :param config: the incoming config dictionary :param parent_base: the name of the parent node, if any :return: a list of expanded, graph-based config items """ output = [] this = {} # We should only have one key, with multiple values, being the # config entries. e.g. (this was checked by config_tree_to_graph) # mkfs: # type: ext4 # label: 1234 assert len(config.items()) == 1 for k, v in config.items(): key = k values = v # If we don't have a base, we take the parent base; first element # can have no base, however. if 'base' not in values: if parent_base is not None: this['base'] = parent_base else: this['base'] = values['base'] # If we don't have a name, it is made up as "key_base" if 'name' not in values: this['name'] = "%s_%s" % (key, this['base']) else: this['name'] = values['name'] # Go through the the values dictionary. Either this is a "plugin" # key that needs to be recursed, or it is a value that is part of # this config entry. for nk, nv in values.items(): if nk == "partitions": # "partitions" is a special key of the "partitioning" # object. It is a list. Each list-entry gets treated # as a top-level entry, so we need to recurse it's # keys. But instead of becoming its own entry in the # graph, it gets attached to the .partitions attribute # of the parent. (see end for example) this['partitions'] = [] for partition in nv: new_part = {} for pk, pv in partition.items(): if is_a_plugin(pk): output.extend( recurse_config({pk: pv}, partition['name'])) else: new_part[pk] = pv new_part['base'] = this['base'] this['partitions'].append(new_part) elif is_a_plugin(nk): # is this key a plugin directive? If so, we recurse # into it. output.extend(recurse_config({nk: nv}, this['name'])) else: # A value entry; just save as part of this entry this[nk] = nv output.append({k: this}) return output def config_tree_to_graph(config): """Turn a YAML config into a graph config Our YAML config is a list of entries. Each Arguments: :parm config: YAML config; either graph or tree :return: graph-based result """ output = [] for entry in config: # Top-level entries should be a dictionary and have a plugin # registered for it if not isinstance(entry, dict): raise BlockDeviceSetupException( "Config entry not a dict: %s" % entry) keys = list(entry.keys()) if len(keys) != 1: raise BlockDeviceSetupException( "Config entry top-level should be a single dict: %s" % entry) if not is_a_plugin(keys[0]): raise BlockDeviceSetupException( "Config entry is not a plugin value: %s" % entry) output.extend(recurse_config(entry)) return output def create_graph(config, default_config, state): """Generate configuration digraph Generate the configuration digraph from the config :param config: graph configuration file :param default_config: default parameters (from --params) :param state: reference to global state dictionary. Passed to :func:`PluginBase.__init__` :return: tuple with the graph object (a :class:`nx.Digraph`), ordered list of :class:`NodeBase` objects """ # This is the directed graph of nodes: each parse method must # add the appropriate nodes and edges. dg = nx.DiGraph() for config_entry in config: # this should have been checked by generate_config assert len(config_entry) == 1 logger.debug("Config entry [%s]", config_entry) cfg_obj_name = list(config_entry.keys())[0] cfg_obj_val = config_entry[cfg_obj_name] # Instantiate a "plugin" object, passing it the # configuration entry # XXX : would a "factory" pattern for plugins, where we # make a method call on an object stevedore has instantiated # be better here? if not is_a_plugin(cfg_obj_name): raise BlockDeviceSetupException( ("Config element [%s] is not implemented" % cfg_obj_name)) plugin = _extensions[cfg_obj_name].plugin assert issubclass(plugin, PluginBase) cfg_obj = plugin(cfg_obj_val, default_config, state) # Ask the plugin for the nodes it would like to insert # into the graph. Some plugins, such as partitioning, # return multiple nodes from one config entry. nodes = cfg_obj.get_nodes() assert isinstance(nodes, list) for node in nodes: # plugins should return nodes... assert isinstance(node, NodeBase) # ensure node names are unique. networkx by default # just appends the attribute to the node dict for # existing nodes, which is not what we want. if node.name in dg.node: raise BlockDeviceSetupException( "Duplicate node name: %s" % (node.name)) logger.debug("Adding %s : %s", node.name, node) dg.add_node(node.name, obj=node) # Now find edges for name, attr in dg.nodes(data=True): obj = attr['obj'] # Unfortunately, we can not determine node edges just from # the configuration file. It's not always simply the # "base:" pointer. So ask nodes for a list of nodes they # want to point to. *mostly* it's just base: ... but # mounting is different. # edges_from are the nodes that point to us # edges_to are the nodes we point to edges_from, edges_to = obj.get_edges() logger.debug("Edges for %s: f:%s t:%s", name, edges_from, edges_to) for edge_from in edges_from: if edge_from not in dg.node: raise BlockDeviceSetupException( "Edge not defined: %s->%s" % (edge_from, name)) dg.add_edge(edge_from, name) for edge_to in edges_to: if edge_to not in dg.node: raise BlockDeviceSetupException( "Edge not defined: %s->%s" % (name, edge_to)) dg.add_edge(name, edge_to) # this can be quite helpful debugging but needs pydotplus which # isn't in requirements. for debugging, do # .tox/py27/bin/pip install pydotplus # DUMP_CONFIG_GRAPH=1 tox -e py27 -- specific_test # dotty /tmp/graph_dump.dot # to see helpful output if 'DUMP_CONFIG_GRAPH' in os.environ: nx.nx_pydot.write_dot(dg, '/tmp/graph_dump.dot') # Topological sort (i.e. create a linear array that satisfies # dependencies) and return the object list call_order_nodes = nx.topological_sort(dg) logger.debug("Call order: %s", list(call_order_nodes)) call_order = [dg.node[n]['obj'] for n in call_order_nodes] return dg, call_order # # On partitioning: objects # # To be concrete -- # # partitioning: # base: loop0 # name: mbr # partitions: # - name: partition1 # foo: bar # mkfs: # type: xfs # mount: # mount_point: / # # gets turned into the following graph: # # partitioning: # partitions: # - name: partition1 # base: image0 # foo: bar # # mkfs: # base: partition1 # name: mkfs_partition1 # type: xfs # # mount: # base: mkfs_partition1 # name: mount_mkfs_partition1 # mount_point: / diskimage-builder-2.11.0/diskimage_builder/block_device/exception.py000066400000000000000000000011641324273057700256140ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. class BlockDeviceSetupException(Exception): """Generic exception""" diskimage-builder-2.11.0/diskimage_builder/block_device/level0/000077500000000000000000000000001324273057700244315ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/level0/__init__.py000066400000000000000000000000001324273057700265300ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/level0/localloop.py000066400000000000000000000112761324273057700267760ustar00rootroot00000000000000# Copyright 2016 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import logging import os import subprocess from diskimage_builder.block_device.exception import \ BlockDeviceSetupException from diskimage_builder.block_device.plugin import NodeBase from diskimage_builder.block_device.plugin import PluginBase from diskimage_builder.block_device.utils import parse_abs_size_spec logger = logging.getLogger(__name__) def image_create(filename, size): logger.info("Create image file [%s]", filename) with open(filename, "w") as fd: fd.seek(size - 1) fd.write("\0") def image_delete(filename): logger.info("Remove image file [%s]", filename) os.remove(filename) def loopdev_attach(filename): logger.info("loopdev attach") logger.debug("Calling [sudo losetup --show -f %s]", filename) subp = subprocess.Popen(["sudo", "losetup", "--show", "-f", filename], stdout=subprocess.PIPE) rval = subp.wait() if rval == 0: # [:-1]: Cut of the newline block_device = subp.stdout.read()[:-1].decode("utf-8") logger.info("New block device [%s]", block_device) return block_device else: logger.error("losetup failed") raise BlockDeviceSetupException("losetup failed") def loopdev_detach(loopdev): logger.info("loopdev detach") # loopback dev may be tied up a bit by udev events triggered # by partition events for try_cnt in range(10, 1, -1): logger.debug("Calling [sudo losetup -d %s]", loopdev) subp = subprocess.Popen(["sudo", "losetup", "-d", loopdev]) rval = subp.wait() if rval == 0: logger.info("Successfully detached [%s]", loopdev) return 0 else: logger.error("loopdev detach failed") # Do not raise an error - maybe other cleanup methods # can at least do some more work. logger.debug("Gave up trying to detach [%s]", loopdev) return rval class LocalLoopNode(NodeBase): """Level0: Local loop image device handling. This class handles local loop devices that can be used for VM image installation. """ def __init__(self, config, default_config, state): logger.debug("Creating LocalLoop object; config [%s] " "default_config [%s]", config, default_config) super(LocalLoopNode, self).__init__(config['name'], state) if 'size' in config: self.size = parse_abs_size_spec(config['size']) logger.debug("Image size [%s]", self.size) else: self.size = parse_abs_size_spec(default_config['image-size']) logger.debug("Using default image size [%s]", self.size) if 'directory' in config: self.image_dir = config['directory'] else: self.image_dir = default_config['image-dir'] self.filename = os.path.join(self.image_dir, self.name + ".raw") def get_edges(self): """Because this is created without base, there are no edges.""" return ([], []) def create(self): logger.debug("[%s] Creating loop on [%s] with size [%d]", self.name, self.filename, self.size) self.add_rollback(image_delete, self.filename) image_create(self.filename, self.size) block_device = loopdev_attach(self.filename) self.add_rollback(loopdev_detach, block_device) if 'blockdev' not in self.state: self.state['blockdev'] = {} self.state['blockdev'][self.name] = {"device": block_device, "image": self.filename} logger.debug("Created loop name [%s] device [%s] image [%s]", self.name, block_device, self.filename) return def umount(self): loopdev_detach(self.state['blockdev'][self.name]['device']) def delete(self): image_delete(self.state['blockdev'][self.name]['image']) class LocalLoop(PluginBase): def __init__(self, config, defaults, state): super(LocalLoop, self).__init__() self.node = LocalLoopNode(config, defaults, state) def get_nodes(self): return [self.node] diskimage-builder-2.11.0/diskimage_builder/block_device/level1/000077500000000000000000000000001324273057700244325ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/level1/__init__.py000066400000000000000000000000001324273057700265310ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/level1/lvm.py000066400000000000000000000333551324273057700256130ustar00rootroot00000000000000# Copyright 2017 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import subprocess from diskimage_builder.block_device.exception \ import BlockDeviceSetupException from diskimage_builder.block_device.plugin import NodeBase from diskimage_builder.block_device.plugin import PluginBase from diskimage_builder.block_device.utils import exec_sudo logger = logging.getLogger(__name__) # # LVM # --- # # The LVM config has three required keys; pvs, vgs and lvs # # lvm: -> LVSNode # pvs: -> PvsNode # lvs: -> LvsNode # vgs: -> VgsNode # # The LVMPlugin will verify this and build nodes into the # configuration graph. # # As described below, a LVSNode is created for synchronisation # purposes. Thus if you had something like two partitions that became # two physical-volumes (pv1 & pv2), that you then combine into a # single volume group (vg) and then create several logical volumes # (lv1, lv2, lv3) your graph would end up looking like: # # partition1 partition2 # | | # ---> LVSNode <--+ # | # +------+------+ # v v # pv1 pv2 # | | # +--> vg <---+ # | # +-----+-----+ # v v v # lv1 lv2 lv3 # # After the create() call on the LVSNode object, the entire LVM setup # would actually be complete. The other nodes are all just # place-holders, and are used for further ordering (for example, the # fs creation & mounting should depend on the logical volume nodes). # For this reason, their create() calls are blank. However, for code # organisational purposes they have a private _create() and _cleanup() # call that is driven by the LVSNode object. class PvsNode(NodeBase): def __init__(self, name, state, base, options): """Physical volume This is a placeholder node for the LVM physical volumes. Arguments: :param name: Name of this node :param state: global state pointer :param base: Parent partition :param options: config options """ super(PvsNode, self).__init__(name, state) self.base = base self.options = options def _create(self): # the underlying device path of our parent was previously # recorded into the state during blockdev creation; look it # up. phys_dev = self.state['blockdev'][self.base]['device'] cmd = ["pvcreate"] cmd.append(phys_dev) if self.options: cmd.extend(self.options) logger.debug("Creating pv command [%s]", cmd) exec_sudo(cmd) # save state if 'pvs' not in self.state: self.state['pvs'] = {} self.state['pvs'][self.name] = { 'opts': self.options, 'device': phys_dev } def get_edges(self): # See LVMNode.get_edges() for how this gets connected return ([], []) def create(self): # see notes in LVMNode object pass class VgsNode(NodeBase): def __init__(self, name, state, base, options): """Volume Group This is a placeholder node for a volume group Arguments: :param name: Name of this node :param state: global state pointer :param base: Parent :class:`PvsNodes` this volume group exists on :param options: extra options passed to the `vgcreate` command """ super(VgsNode, self).__init__(name, state) self.base = base self.options = options def _create(self): # The PV's have saved their actual device name into the state # during their _create(). Look at our base elements and thus # find the underlying device paths in the state. pvs_devs = [] for pv in self.base: pvs_dev = self.state['pvs'][pv]['device'] pvs_devs.append(pvs_dev) cmd = ["vgcreate", ] cmd.append(self.name) cmd.extend(pvs_devs) if self.options: cmd.extend(self.options) logger.debug("Creating vg command [%s]", cmd) exec_sudo(cmd) # save state if 'vgs' not in self.state: self.state['vgs'] = {} self.state['vgs'][self.name] = { 'opts': self.options, 'devices': self.base, } def _cleanup(self): exec_sudo(['vgchange', '-an', self.name]) def get_edges(self): # self.base is already a list, per the config. There might be # multiple pv parents here. edge_from = self.base edge_to = [] return (edge_from, edge_to) def create(self): # see notes in LVMNode object pass class LvsNode(NodeBase): def __init__(self, name, state, base, options, size, extents): """Logical Volume This is a placeholder node for a logical volume Arguments: :param name: Name of this node :param state: global state pointer :param base: the parent volume group :param options: options passed to lvcreate :param size: size of the LV, in MB (this or extents must be provided) :param extents: size of the LV in extents """ super(LvsNode, self).__init__(name, state) self.base = base self.options = options self.size = size self.extents = extents def _create(self): cmd = ["lvcreate", ] cmd.extend(['--name', self.name]) if self.size: cmd.extend(['-L', self.size]) elif self.extents: cmd.extend(['-l', self.extents]) if self.options: cmd.extend(self.options) cmd.append(self.base) logger.debug("Creating lv command [%s]", cmd) exec_sudo(cmd) # save state self.state['blockdev'][self.name] = { 'vgs': self.base, 'size': self.size, 'extents': self.extents, 'opts': self.options, 'device': '/dev/mapper/%s-%s' % (self.base, self.name) } def _cleanup(self): exec_sudo(['lvchange', '-an', '/dev/%s/%s' % (self.base, self.name)]) def get_edges(self): edge_from = [self.base] edge_to = [] return (edge_from, edge_to) def create(self): # see notes in LVMNode object pass class LVMNode(NodeBase): def __init__(self, name, state, pvs, lvs, vgs): """LVM Driver Node This is the "global" node where all LVM operations are driven from. In the node graph, the LVM physical-volumes depend on this node. This node then depends on the devices that the PV's require. This node incorporates *all* LVM setup; i.e. after the create() call here we have created all pv's, lv's and vg. The Node objects in the graph are therefore just dependency place holders whose create() call does nothing. Arguments: :param name: name of this node :param state: global state pointer :param pvs: A list of :class:`PvsNode` objects :param lvs: A list of :class:`LvsNode` objects :param vgs: A list of :class:`VgsNode` objects """ super(LVMNode, self).__init__(name, state) self.pvs = pvs self.lvs = lvs self.vgs = vgs def get_edges(self): # This node requires the physical device(s), which is # recorded in the "base" argument of the PV nodes. pvs = [] for pv in self.pvs: pvs.append(pv.base) edge_from = set(pvs) # The PV nodes should then depend on us. i.e., we just made # this node a synchronisation point edge_to = [pv.name for pv in self.pvs] return (edge_from, edge_to) def create(self): # Run through pvs->vgs->lvs and create them # XXX: we could theoretically get this same info from walking # the graph of our children nodes? Would that be helpful in # any way? for pvs in self.pvs: pvs._create() for vgs in self.vgs: vgs._create() for lvs in self.lvs: lvs._create() def cleanup(self): for lvs in self.lvs: lvs._cleanup() for vgs in self.vgs: vgs._cleanup() exec_sudo(['udevadm', 'settle']) class LVMCleanupNode(NodeBase): def __init__(self, name, state, pvs): """Cleanup Node for LVM Information about the PV, VG and LV is typically cached in lvmetad. Even after removing PV device and partitions this data is not automatically updated which leads to a couple of problems. the 'pvscan --cache' scans the available disks and updates the cache. This must be called after the cleanup of the containing block device is done. """ super(LVMCleanupNode, self).__init__(name, state) self.pvs = pvs def create(self): # This class is used for cleanup only pass def cleanup(self): try: exec_sudo(['pvscan', '--cache']) except subprocess.CalledProcessError as cpe: logger.debug("pvscan call result [%s]", cpe) def get_edges(self): # This node depends on all physical device(s), which is # recorded in the "base" argument of the PV nodes. edge_to = set() for pv in self.pvs: edge_to.add(pv.base) return ([], edge_to) class LVMPlugin(PluginBase): def _config_error(self, msg): raise BlockDeviceSetupException(msg) def __init__(self, config, defaults, state): """Build LVM nodes This reads the "lvm:" config stanza, validates it and produces the PV, VG and LV nodes. These are all synchronised via a LVMNode as described above. Arguments: :param config: "lvm" configuration dictionary :param defaults: global defaults dictionary :param state: global state reference """ super(LVMPlugin, self).__init__() # note lvm: doesn't require a base ... the base is the # physical devices the "pvs" nodes are made on. if 'name' not in config: self._config_error("Lvm config requires 'name'") if 'pvs' not in config: self._config_error("Lvm config requires a 'pvs'") if 'vgs' not in config: self._config_error("Lvm config needs 'vgs'") if 'lvs' not in config: self._config_error("Lvm config needs 'lvs'") # create physical volume nodes self.pvs = [] self.pvs_keys = [] for pvs_cfg in config['pvs']: if 'name' not in pvs_cfg: self._config_error("Missing 'name' in pvs config") if 'base' not in pvs_cfg: self._config_error("Missing 'base' in pvs_config") pvs_item = PvsNode(pvs_cfg['name'], state, pvs_cfg['base'], pvs_cfg.get('options')) self.pvs.append(pvs_item) # create volume group nodes self.vgs = [] self.vgs_keys = [] for vgs_cfg in config['vgs']: if 'name' not in vgs_cfg: self._config_error("Missing 'name' in vgs config") if 'base' not in vgs_cfg: self._config_error("Missing 'base' in vgs config") # Ensure we have a valid PVs backing this VG for pvs in vgs_cfg['base']: if not any(pv.name == pvs for pv in self.pvs): self._config_error("base:%s in vgs does not " "match a valid pvs" % pvs) vgs_item = VgsNode(vgs_cfg['name'], state, vgs_cfg['base'], vgs_cfg.get('options', None)) self.vgs.append(vgs_item) # create logical volume nodes self.lvs = [] for lvs_cfg in config['lvs']: if 'name' not in lvs_cfg: self._config_error("Missing 'name' in lvs config") if 'base' not in lvs_cfg: self._config_error("Missing 'base' in lvs config") if 'size' not in lvs_cfg and 'extents' not in lvs_cfg: self._config_error("Missing 'size' or 'extents' in lvs config") # ensure this logical volume has a valid volume group base if not any(vg.name == lvs_cfg['base'] for vg in self.vgs): self._config_error("base:%s in lvs does not match a valid vg" % lvs_cfg['base']) lvs_item = LvsNode(lvs_cfg['name'], state, lvs_cfg['base'], lvs_cfg.get('options', None), lvs_cfg.get('size', None), lvs_cfg.get('extents', None)) self.lvs.append(lvs_item) # create the "driver" node self.lvm_node = LVMNode(config['name'], state, self.pvs, self.lvs, self.vgs) self.lvm_cleanup_node = LVMCleanupNode( config['name'] + "-CLEANUP", state, self.pvs) def get_nodes(self): # the nodes for insertion into the graph are all of the pvs, # vgs and lvs nodes we have created above, the root node and # the cleanup node. return self.pvs + self.vgs + self.lvs \ + [self.lvm_node, self.lvm_cleanup_node] diskimage-builder-2.11.0/diskimage_builder/block_device/level1/mbr.py000066400000000000000000000361241324273057700255720ustar00rootroot00000000000000# Copyright 2016 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import logging import os import random from struct import pack logger = logging.getLogger(__name__) # Details of the MBR object itself can be found in the inline # documentation. # # General design and implementation remarks: # o Because the whole GNU parted and co. (e.g. the python-parted that # is based on GNU parted) cannot be used because of the license: # everything falls under GPL2 (not LGPL2!) and therefore does not # fit into the Apache License here. # o It looks that there is no real alternative available (2016-06). # o The interface of python-parted is not that simple to handle - and # the initial try to use GNU (python-)parted was not that much # easier and shorter than this approach. # o When using tools (like fdisk or parted) they try to optimize the # alignment of partitions based on the data found on the host # system. These might be misleading and might lead to (very) poor # performance. # o These ready-to-use tools typically also change the CHS layout # based on the disk size. In case that the disk is enlarged (which # is a normal use case for golden images), the CHS layout of the # disk changes for those tools (and is not longer correct). # In the DIB implementation the CHS are chosen that way, that also # for very small disks the maximum heads/cylinder and sectors/track # is used: even if the disk size in increased, the CHS numbers will # not change. # o In the easy and straight forward way when only using one # partition, exactly 40 bytes (!) must be written - and the biggest # part of this data is fixed (same in all cases). # # Limitations and Incompatibilities # o With the help of this class it is possible to create an # arbitrarily number of extended partitions (tested with over 1000). # o There are limitations and shortcomings in the OS and in tools # handling these partitions. # o Under Linux the loop device is able to handle a limited number of # partitions. The module parameter max_loop can be set - the maximum # number might vary depending on the distribution and kernel build. # o Under Linux fdisk is able to handle 'only' 60 partitions. Only # those are listed, can be changed or written. # o Under Linux GNU parted can handle about 60 partitions. # # Be sure only to pass in the number of partitions that the host OS # and target OS are able to handle. class MBR(object): """MBR Disk / Partition Table Layout Primary partitions are created first - and must also be passed in first. The extended partition layout is done in the way, that there is one entry in the MBR (the last) that uses the whole disk. EBR (extended boot records) are used to describe the partitions themselves. This has the advantage, that the same procedure can be used for all partitions and arbitrarily many partitions can be created in the same way (the EBR is placed as block 0 in each partition itself). In conjunction with a fixed and 'fits all' partition alignment the major design focus is maximum performance for the installed image (vs. minimal size). Because of the chosen default alignment of 1MiB there will be (1MiB - 512B) unused disk space for the MBR and also the same size unused in every partition. Assuming that 512 byte blocks are used, the resulting layout for extended partitions looks like (blocks offset in extended partition given): ======== ============================================== Offset Description ======== ============================================== 0 MBR - 2047 blocks unused 2048 EBR for partition 1 - 2047 blocks unused 4096 Start of data for partition 1 ... ... X EBR for partition N - 2047 blocks unused X+2048 Start of data for partition N ======== ============================================== Direct (native) writing of MBR, EBR (partition table) is implemented - no other partitioning library or tools is used - to be sure to get the correct CHS and alignment for a wide range of host systems. """ # Design & Implementation details: # o A 'block' is a storage unit on disk. It is similar (equal) to a # sector - but with LBA addressing. # o It is assumed that a disk block has that number of bytes bytes_per_sector = 512 # o CHS is the 'good and very old way' specifying blocks. # When passing around these numbers, they are also ordered like 'CHS': # (cylinder, head, sector). # o The computation from LBA to CHS is not unique (it is based # on the 'real' (or assumed) number of heads/cylinder and # sectors/track), these are the assumed numbers. Please note # that these are also the maximum numbers: heads_per_cylinder = 254 sectors_per_track = 63 max_cylinders = 1023 # o There is the need for some offsets that are defined in the # MBR/EBR domain. MBR_offset_disk_id = 440 MBR_offset_signature = 510 MBR_offset_first_partition_table_entry = 446 MBR_partition_type_extended_chs = 0x5 MBR_partition_type_extended_lba = 0xF MBR_signature = 0xAA55 def __init__(self, name, disk_size, alignment): """Initialize a disk partitioning MBR object. The name is the (existing) name of the disk. The disk_size is the (used) size of the disk. It must be a proper multiple of the disk bytes per sector (currently 512) """ logger.info("Create MBR disk partitioning object") assert disk_size % MBR.bytes_per_sector == 0 self.disk_size = disk_size self.disk_size_in_blocks \ = self.disk_size // MBR.bytes_per_sector self.alignment_blocks = alignment // MBR.bytes_per_sector # Because the extended partitions are a chain of blocks, when # creating a new partition, the reference in the already # existing EBR must be updated. This holds a reference to the # latest EBR. (A special case is the first: when it points to # 0 (MBR) there is no need to update the reference.) self.disk_block_last_ref = 0 self.name = name self.partition_abs_start = None self.partition_abs_next_free = None # Start of partition number self.partition_number = 0 self.primary_partitions_created = 0 self.extended_partitions_created = 0 def __enter__(self): # Open existing file for writing (r+) self.image_fd = open(self.name, "r+b") self.write_mbr() self.write_mbr_signature(0) self.partition_abs_start = self.align(1) self.partition_abs_next_free \ = self.partition_abs_start return self def __exit__(self, exc_type, exc_value, traceback): self.image_fd.flush() os.fsync(self.image_fd.fileno()) self.image_fd.close() def lba2chs(self, lba): """Converts a LBA block number to CHS If the LBA block number is bigger than the max (1023, 63, 254) the maximum is returned. """ if lba > MBR.heads_per_cylinder * MBR.sectors_per_track \ * MBR.max_cylinders: return MBR.max_cylinders, MBR.heads_per_cylinder, \ MBR.sectors_per_track cylinder = lba // (MBR.heads_per_cylinder * MBR.sectors_per_track) head = (lba // MBR.sectors_per_track) % MBR.heads_per_cylinder sector = (lba % MBR.sectors_per_track) + 1 logger.debug("Convert LBA to CHS [%d] -> [%d, %d, %d]", lba, cylinder, head, sector) return cylinder, head, sector def encode_chs(self, cylinders, heads, sectors): """Encodes a CHS triple into disk format""" # Head - nothing to convert assert heads <= MBR.heads_per_cylinder eh = heads # Sector assert sectors <= MBR.sectors_per_track es = sectors # top two bits are set in cylinder conversion # Cylinder assert cylinders <= MBR.max_cylinders ec = cylinders % 256 # lower part hc = cylinders // 4 # extract top two bits and es = es | hc # pass them into the top two bits of the sector logger.debug("Encode CHS to disk format [%d %d %d] " "-> [%02x %02x %02x]", cylinders, heads, sectors, eh, es, ec) return eh, es, ec def write_mbr(self): """Write MBR This method writes the MBR to disk. It creates a random disk id as well that it creates the extended partition (as first partition) which uses the whole disk. """ disk_id = random.randint(0, 0xFFFFFFFF) self.image_fd.seek(MBR.MBR_offset_disk_id) self.image_fd.write(pack(" 0: raise RuntimeError("All primary partitions must be " "given first") if primaryflag: return self.add_primary_partition(bootflag, size, ptype) if self.extended_partitions_created == 0: # When this is the first extended partition, the extended # partition entry has to be written. self.partition_abs_start = self.partition_abs_next_free self.write_partition_entry( False, 0, self.partition_number, MBR.MBR_partition_type_extended_lba, self.partition_abs_next_free, self.disk_size_in_blocks - self.partition_abs_next_free) self.partition_number = 4 return self.add_extended_partition(bootflag, size, ptype) def free(self): """Returns the free (not yet partitioned) size""" return self.disk_size \ - (self.partition_abs_next_free + self.align(1)) \ * MBR.bytes_per_sector diskimage-builder-2.11.0/diskimage_builder/block_device/level1/partition.py000066400000000000000000000047141324273057700270230ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import logging from diskimage_builder.block_device.exception import \ BlockDeviceSetupException from diskimage_builder.block_device.plugin import NodeBase logger = logging.getLogger(__name__) class PartitionNode(NodeBase): flag_boot = 1 flag_primary = 2 def __init__(self, config, state, parent, prev_partition): super(PartitionNode, self).__init__(config['name'], state) self.base = config['base'] self.partitioning = parent self.prev_partition = prev_partition self.flags = set() if 'flags' in config: for f in config['flags']: if f == 'boot': self.flags.add(self.flag_boot) elif f == 'primary': self.flags.add(self.flag_primary) else: raise BlockDeviceSetupException("Unknown flag: %s" % f) if 'size' not in config: raise BlockDeviceSetupException("No size in partition" % self.name) self.size = config['size'] self.ptype = int(config['type'], 16) if 'type' in config else 0x83 def get_flags(self): return self.flags def get_size(self): return self.size def get_type(self): return self.ptype def get_edges(self): edge_from = [self.base] edge_to = [] if self.prev_partition is not None: edge_from.append(self.prev_partition.name) return (edge_from, edge_to) # These all call back to the parent "partitioning" object to do # the real work. Every node calls it, but only one will succeed; # see the gating we do in the parent function. # # XXX: A better model here would be for the parent object to a # real node in the config graph, so it's create() gets called. # These can then just be stubs. def create(self): self.partitioning.create() def cleanup(self): self.partitioning.cleanup() diskimage-builder-2.11.0/diskimage_builder/block_device/level1/partitioning.py000066400000000000000000000160401324273057700275140ustar00rootroot00000000000000# Copyright 2016 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import logging import os from diskimage_builder.block_device.exception import \ BlockDeviceSetupException from diskimage_builder.block_device.level1.mbr import MBR from diskimage_builder.block_device.level1.partition import PartitionNode from diskimage_builder.block_device.plugin import PluginBase from diskimage_builder.block_device.utils import exec_sudo from diskimage_builder.block_device.utils import parse_abs_size_spec from diskimage_builder.block_device.utils import parse_rel_size_spec logger = logging.getLogger(__name__) class Partitioning(PluginBase): def __init__(self, config, default_config, state): logger.debug("Creating Partitioning object; config [%s]", config) super(Partitioning, self).__init__() # Unlike other PluginBase we are somewhat persistent, as the # partition nodes call back to us (see create() below). We # need to keep this reference. self.state = state # Because using multiple partitions of one base is done # within one object, there is the need to store a flag if the # creation of the partitions was already done. self.already_created = False self.already_cleaned = False # Parameter check if 'base' not in config: raise BlockDeviceSetupException("Partitioning config needs 'base'") self.base = config['base'] if 'partitions' not in config: raise BlockDeviceSetupException( "Partitioning config needs 'partitions'") if 'label' not in config: raise BlockDeviceSetupException( "Partitioning config needs 'label'") self.label = config['label'] if self.label not in ("mbr", ): raise BlockDeviceSetupException("Label must be 'mbr'") # It is VERY important to get the alignment correct. If this # is not correct, the disk performance might be very poor. # Example: In some tests a 'off by one' leads to a write # performance of 30% compared to a correctly aligned # partition. # The problem for DIB is, that it cannot assume that the host # system uses the same IO sizes as the target system, # therefore here a fixed approach (as used in all modern # systems with large disks) is used. The partitions are # aligned to 1MiB (which are about 2048 times 512 bytes # blocks) self.align = 1024 * 1024 # 1MiB as default if 'align' in config: self.align = parse_abs_size_spec(config['align']) self.partitions = [] prev_partition = None for part_cfg in config['partitions']: np = PartitionNode(part_cfg, state, self, prev_partition) self.partitions.append(np) prev_partition = np def get_nodes(self): # return the list of partitions return self.partitions def _size_of_block_dev(self, dev): with open(dev, "r") as fd: fd.seek(0, 2) return fd.tell() # not this is NOT a node and this is not called directly! The # create() calls in the partition nodes this plugin has # created are calling back into this. def create(self): # This is a bit of a hack. Each of the partitions is actually # in the graph, so for every partition we get a create() call # as the walk happens. But we only need to create the # partition table once... if self.already_created: logger.info("Not creating the partitions a second time.") return self.already_created = True # the raw file on disk image_path = self.state['blockdev'][self.base]['image'] # the /dev/loopX device of the parent device_path = self.state['blockdev'][self.base]['device'] logger.info("Creating partition on [%s] [%s]", self.base, image_path) assert self.label == 'mbr' disk_size = self._size_of_block_dev(image_path) with MBR(image_path, disk_size, self.align) as part_impl: for part_cfg in self.partitions: part_name = part_cfg.get_name() part_bootflag = PartitionNode.flag_boot \ in part_cfg.get_flags() part_primary = PartitionNode.flag_primary \ in part_cfg.get_flags() part_size = part_cfg.get_size() part_free = part_impl.free() part_type = part_cfg.get_type() logger.debug("Not partitioned space [%d]", part_free) part_size = parse_rel_size_spec(part_size, part_free)[1] part_no \ = part_impl.add_partition(part_primary, part_bootflag, part_size, part_type) logger.debug("Create partition [%s] [%d]", part_name, part_no) # We're going to mount all partitions with kpartx # below once we're done. So the device this partition # will be seen at becomes "/dev/mapper/loop0pX" assert device_path[:5] == "/dev/" partition_device_name = "/dev/mapper/%sp%d" % \ (device_path[5:], part_no) self.state['blockdev'][part_name] \ = {'device': partition_device_name} # "saftey sync" to make sure the partitions are written exec_sudo(["sync"]) # now all the partitions are created, get device-mapper to # mount them if not os.path.exists("/.dockerenv"): exec_sudo(["kpartx", "-avs", device_path]) else: # If running inside Docker, make our nodes manually, # because udev will not be working. kpartx cannot run in # sync mode in docker. exec_sudo(["kpartx", "-av", device_path]) exec_sudo(["dmsetup", "--noudevsync", "mknodes"]) return def cleanup(self): # remove the partition mappings made for the parent # block-device by create() above. this is called from the # child PartitionNode umount/delete/cleanup. Thus every # partition calls it, but we only want to do it once and our # gate. if not self.already_cleaned: self.already_cleaned = True exec_sudo(["kpartx", "-d", self.state['blockdev'][self.base]['device']]) diskimage-builder-2.11.0/diskimage_builder/block_device/level2/000077500000000000000000000000001324273057700244335ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/level2/__init__.py000066400000000000000000000000001324273057700265320ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/level2/mkfs.py000066400000000000000000000121201324273057700257410ustar00rootroot00000000000000# Copyright 2017 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import logging import uuid from diskimage_builder.block_device.exception \ import BlockDeviceSetupException from diskimage_builder.block_device.plugin import NodeBase from diskimage_builder.block_device.plugin import PluginBase from diskimage_builder.block_device.utils import exec_sudo logger = logging.getLogger(__name__) # There is the need to check the length of the label of # the filesystem. The maximum length depends on the used filesystem. # This map provides information about the maximum label length. file_system_max_label_length = { "ext2": 16, "ext3": 16, "ext4": 16, "xfs": 12, "vfat": 11 } class FilesystemNode(NodeBase): def __init__(self, config, state): logger.debug("Create filesystem object; config [%s]", config) super(FilesystemNode, self).__init__(config['name'], state) # Parameter check (mandatory) for pname in ['base', 'type']: if pname not in config: raise BlockDeviceSetupException( "Mkfs config needs [%s]" % pname) setattr(self, pname, config[pname]) # Parameter check (optional) for pname in ['label', 'opts', 'uuid']: setattr(self, pname, config[pname] if pname in config else None) if self.label is None: self.label = self.name # ensure we don't already have a fs with this label ... they # all must be unique. if 'fs_labels' in self.state: if self.label in self.state['fs_labels']: raise BlockDeviceSetupException( "File system label [%s] used more than once" % self.label) self.state['fs_labels'].append(self.label) else: self.state['fs_labels'] = [self.label] if self.type in file_system_max_label_length: if file_system_max_label_length[self.type] < len(self.label): raise BlockDeviceSetupException( "Label [{label}] too long for filesystem [{type}]: " "{len} > {max_len}".format(**{ 'label': self.label, 'type': self.type, 'len': len(self.label), 'max_len': file_system_max_label_length[self.type]})) else: logger.warning("Length of label [%s] cannot be checked for " "filesystem [%s]: unknown max length", self.label, self.type) logger.warning("Continue - but this might lead to an error") if self.opts is not None: self.opts = self.opts.strip().split(' ') if self.uuid is None: self.uuid = str(uuid.uuid4()) logger.debug("Filesystem created [%s]", self) def get_edges(self): edge_from = [self.base] edge_to = [] return (edge_from, edge_to) def create(self): cmd = ["mkfs"] cmd.extend(['-t', self.type]) if self.opts: cmd.extend(self.opts) if self.type in ('vfat', 'fat'): cmd.extend(["-n", self.label]) else: cmd.extend(["-L", self.label]) if self.type in ('ext2', 'ext3', 'ext4'): cmd.extend(['-U', self.uuid]) elif self.type == 'xfs': cmd.extend(['-m', "uuid=%s" % self.uuid]) else: logger.warning("UUID will not be written for fs type [%s]", self.type) if self.type in ('ext2', 'ext3', 'ext4', 'xfs'): cmd.append('-q') if 'blockdev' not in self.state: self.state['blockdev'] = {} device = self.state['blockdev'][self.base]['device'] cmd.append(device) logger.debug("Creating fs command [%s]", cmd) exec_sudo(cmd) if 'filesys' not in self.state: self.state['filesys'] = {} self.state['filesys'][self.name] \ = {'uuid': self.uuid, 'label': self.label, 'fstype': self.type, 'opts': self.opts, 'device': device} class Mkfs(PluginBase): """Create a file system This block device module handles creating different file systems. """ def __init__(self, config, defaults, state): super(Mkfs, self).__init__() self.filesystems = {} fs = FilesystemNode(config, state) self.filesystems[fs.get_name()] = fs def get_nodes(self): nodes = [] for _, fs in self.filesystems.items(): nodes.append(fs) return nodes diskimage-builder-2.11.0/diskimage_builder/block_device/level3/000077500000000000000000000000001324273057700244345ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/level3/__init__.py000066400000000000000000000000001324273057700265330ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/level3/mount.py000066400000000000000000000151401324273057700261510ustar00rootroot00000000000000# Copyright 2017 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import functools import logging import os from diskimage_builder.block_device.exception \ import BlockDeviceSetupException from diskimage_builder.block_device.plugin import NodeBase from diskimage_builder.block_device.plugin import PluginBase from diskimage_builder.block_device.utils import exec_sudo logger = logging.getLogger(__name__) class MountPointNode(NodeBase): def __init__(self, mount_base, config, state): super(MountPointNode, self).__init__(config['name'], state) # Parameter check self.mount_base = mount_base for pname in ['base', 'mount_point']: if pname not in config: raise BlockDeviceSetupException( "MountPoint config needs [%s]" % pname) setattr(self, pname, config[pname]) logger.debug("MountPoint created [%s]", self) def get_edges(self): """Insert all edges The dependency edge is created in all cases from the base element (typically a mkfs) and, if this is not the 'first' mount-point, an edge is created from the mount-point before in "sorted order" (see :func:`sort_mount_points`). This ensures that during mounting (and umounting) the globally correct order is used. """ edge_from = [] edge_to = [] # should have been added by __init__... assert 'sorted_mount_points' in self.state sorted_mount_points = self.state['sorted_mount_points'] # If we are not first, add our parent in the global dependency # list. sorted_mount_points is tuples (mount_point, node_name). # find ourselves in the mount_points, and our parent node # is one before us in node_name list. mount_points = [x[0] for x in sorted_mount_points] node_name = [x[1] for x in sorted_mount_points] mpi = mount_points.index(self.mount_point) if mpi > 0: dep = node_name[mpi - 1] edge_from.append(dep) edge_from.append(self.base) return (edge_from, edge_to) def create(self): logger.debug("mount called [%s]", self.mount_point) rel_mp = self.mount_point if self.mount_point[0] != '/' \ else self.mount_point[1:] mount_point = os.path.join(self.mount_base, rel_mp) if not os.path.exists(mount_point): # Need to sudo this because of permissions in the new # file system tree. exec_sudo(['mkdir', '-p', mount_point]) logger.info("Mounting [%s] to [%s]", self.name, mount_point) exec_sudo(["mount", self.state['filesys'][self.base]['device'], mount_point]) if 'mount' not in self.state: self.state['mount'] = {} self.state['mount'][self.mount_point] \ = {'name': self.name, 'base': self.base, 'path': mount_point} if 'mount_order' not in self.state: self.state['mount_order'] = [] self.state['mount_order'].append(self.mount_point) def umount(self): logger.info("Called for [%s]", self.name) # Before calling umount, call 'fstrim' on suitable mounted # file systems. This discards unused blocks from the mounted # file system and therefore decreases the resulting image # size. # # A race condition can occur when trying to fstrim immediately # after deleting a file resulting in that free space not being # reclaimed. Calling sync before fstrim is a workaround for # this behaviour. # https://lists.gnu.org/archive/html/qemu-devel/2014-03/msg02978.html exec_sudo(["sync"]) if self.state['filesys'][self.base]['fstype'] != 'vfat': exec_sudo(["fstrim", "--verbose", self.state['mount'][self.mount_point]['path']]) exec_sudo(["umount", self.state['mount'][self.mount_point]['path']]) def delete(self): self.umount() def cmp_mount_order(this, other): """Sort comparision function for mount-point sorting See if ``this`` comes before ``other`` in mount-order list. In words: if the other mount-point has us as it's parent, we come before it (are less than it). e.g. ``/var < /var/log < /var/log/foo`` :param this: tuple of mount_point, node name :param other: tuple of mount_point, node name :returns int: cmp value """ # sort is only based on the mount_point. this, _ = this other, _ = other if this == other: return 0 if other.startswith(this): return -1 else: return 1 class Mount(PluginBase): def __init__(self, config, defaults, state): super(Mount, self).__init__() if 'mount-base' not in defaults: raise BlockDeviceSetupException( "Mount default config needs 'mount-base'") self.node = MountPointNode(defaults['mount-base'], config, state) # save this new node to the global mount-point list and # re-order it to keep it in mount-order. Used in get_edges() # to ensure we build the mount graph in order # # note we can't just put the MountPointNode into the state, # because it's not json serialisable and we still dump the # state to json. that's why we have this (mount_point, name) # tuples and sorting trickery sorted_mount_points = state.get('sorted_mount_points', []) mount_points = [mp for mp, name in sorted_mount_points] if self.node.mount_point in mount_points: raise BlockDeviceSetupException( "Mount point [%s] specified more than once" % self.node.mount_point) sorted_mount_points.append((self.node.mount_point, self.node.name)) sorted_mount_points.sort(key=functools.cmp_to_key(cmp_mount_order)) # Save the state if it's new (otherwise this is idempotent update) state['sorted_mount_points'] = sorted_mount_points logger.debug("Ordered mounts now: %s", sorted_mount_points) def get_nodes(self): return [self.node] diskimage-builder-2.11.0/diskimage_builder/block_device/level4/000077500000000000000000000000001324273057700244355ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/level4/__init__.py000066400000000000000000000000001324273057700265340ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/level4/fstab.py000066400000000000000000000034511324273057700261110ustar00rootroot00000000000000# Copyright 2017 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import logging from diskimage_builder.block_device.plugin import NodeBase from diskimage_builder.block_device.plugin import PluginBase logger = logging.getLogger(__name__) class FstabNode(NodeBase): def __init__(self, config, state): super(FstabNode, self).__init__(config['name'], state) self.base = config['base'] self.options = config.get('options', 'defaults') self.dump_freq = config.get('dump-freq', 0) self.fsck_passno = config.get('fsck-passno', 2) def get_edges(self): edge_from = [self.base] edge_to = [] return (edge_from, edge_to) def create(self): logger.debug("fstab create called [%s]", self.name) if 'fstab' not in self.state: self.state['fstab'] = {} self.state['fstab'][self.base] = { 'name': self.name, 'base': self.base, 'options': self.options, 'dump-freq': self.dump_freq, 'fsck-passno': self.fsck_passno } class Fstab(PluginBase): def __init__(self, config, defaults, state): super(Fstab, self).__init__() self.node = FstabNode(config, state) def get_nodes(self): return [self.node] diskimage-builder-2.11.0/diskimage_builder/block_device/plugin.py000066400000000000000000000147511324273057700251220ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import abc import logging import six # # Plugins convert configuration entries into graph nodes ready for # processing. This defines the abstract classes for both. # logger = logging.getLogger(__name__) @six.add_metaclass(abc.ABCMeta) class NodeBase(object): """A configuration node entry This is the main driver class for dib-block-device operation. The final operations graph is composed of instantiations of this class. The graph undergoes a topological sort (i.e. is linearised in dependency order) and each node has :func:`create` called in order to perform its operations. Every node has a unique string ``name``. This is its key in the graph and used for edge relationships. Implementations must ensure they initalize it; e.g. .. code-block:: python class FooNode(NodeBase): def __init__(name, arg1, ...): super(FooNode, self).__init__(name) """ def __init__(self, name, state): self.name = name self.state = state self.rollbacks = [] def get_name(self): return self.name def add_rollback(self, func, *args, **kwargs): """Add a call for rollback Functions registered with this method will be called in reverse-order in the case of failures during :func:`Nodebase.create`. :param func: function to call :param args: arguments :param kwargs: keyword arguments :return: None """ self.rollbacks.append((func, args, kwargs)) def rollback(self): """Initiate rollback Call registered rollback in reverse order. This method is called by the driver in the case of failures during :func:`Nodebase.create`. :return None: """ # XXX: maybe ignore SystemExit so we always continue? logger.debug("Calling rollback for %s", self.name) for func, args, kwargs in reversed(self.rollbacks): func(*args, **kwargs) @abc.abstractmethod def get_edges(self): """Return the dependencies/edges for this node This function will be called after all nodes are created (this is because some plugins need to know the global state of all nodes to decide their dependencies). This function returns a tuple with two lists * ``edges_from`` : a list of node names that point to us * ``edges_to`` : a list of node names we point to In most cases, node creation will have saved a single parent that was given in the ``base`` parameter of the configuration. A usual return might look like: .. code-block:: python def get_edges(self): return ( [self.base], [] ) Some nodes (``level0``) don't have a base, however """ return @abc.abstractmethod def create(self): """Main creation driver This is the main driver function. After the graph is linearised, each node has it's :func:`create` function called. :raises Exception: A failure should raise an exception. This will initiate a rollback. See :func:`Nodebase.add_rollback`. :return: None """ return def umount(self): """Umount actions Actions to taken when ``dib-block-device umount`` is called. The nodes are called in the reverse order to :func:`create` :return: None """ return def cleanup(self): """Cleanup actions Actions to taken when ``dib-block-device cleanup`` is called. This is the cleanup path in the *success* case. The nodes are called in the reverse order to :func:`create` :return: None """ return def delete(self): """Cleanup actions Actions to taken when ``dib-block-device delete`` is called. This is the cleanup path in case of a reported external *failure*. The nodes are called in the reverse order to :func:`create` :return: None """ return @six.add_metaclass(abc.ABCMeta) class PluginBase(object): """The base plugin object This is the base plugin object. Plugins are an instantiation of this class. There should be an entry-point (see setup.cfg) defined under ``diskimage_builder.block_device.plugin`` for each plugin, e.g. foo = diskimage_builder.block_device.levelX.foo:Foo A configuration entry in the graph config that matches this entry point will create an instance of this class, e.g. .. code-block:: yaml foo: name: foo_node base: parent_node argument_a: bar argument_b: baz The ``__init__`` function will be passed three arguments: ``config`` The full configuration dictionary for the entry. A unique ``name`` entry can be assumed. In most cases a ``base`` entry will be present giving the parent node (see :func:`NodeBase.get_edges`). ``state`` A reference to the gobal state dictionary. This should be passed to :func:`NodeBase.__init__` on node creation ``defaults`` The global defaults dictionary (see ``--params``) ``get_nodes()`` should return the node object(s) created by the config for insertion into the final configuration graph. In the simplest case, this is probably a single node created during instantiation. e.g. .. code-block:: python class Foo(PluginBase): def __init__(self, config, defaults, state): super(Foo, self).__init__() self.node = FooNode(config.name, state, ...) def get_nodes(self): return [self.node] Some plugins require more, however. """ def __init__(self): pass @abc.abstractmethod def get_nodes(self): """Return nodes created by the plugin :returns: a list of :class:`.NodeBase` objects for insertion into the graph """ return diskimage-builder-2.11.0/diskimage_builder/block_device/tests/000077500000000000000000000000001324273057700244045ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/tests/__init__.py000066400000000000000000000000001324273057700265030ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/000077500000000000000000000000001324273057700256515ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/bad_edge_graph.yaml000066400000000000000000000007111324273057700314270ustar00rootroot00000000000000- local_loop: name: image0 - partitioning: base: image0 name: mbr label: mbr partitions: - flags: [boot, primary] name: root base: image0 size: 100% - mount: base: mkfs_root name: mount_mkfs_root mount_point: / - fstab: base: mount_mkfs_root name: fstab_mount_mkfs_root fsck-passno: 1 options: defaults - mkfs: base: this_is_not_a_node name: mkfs_root type: ext4 diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/bad_plugin.yaml000066400000000000000000000000661324273057700306430ustar00rootroot00000000000000- this_is_not_a_plugin_name: foo: bar baz: moodiskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/cmd_create.yaml000066400000000000000000000001261324273057700306220ustar00rootroot00000000000000- test_a: name: test_node_a - test_b: name: test_node_b base: test_node_adiskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/deep_graph.yaml000066400000000000000000000006721324273057700306400ustar00rootroot00000000000000- local_loop: name: image0 - partitioning: base: image0 name: mbr label: mbr partitions: - flags: [boot, primary] name: root base: image0 size: 100% - mount: base: mkfs_root name: mount_mkfs_root mount_point: / - fstab: base: mount_mkfs_root name: fstab_mount_mkfs_root fsck-passno: 1 options: defaults - mkfs: base: root name: mkfs_root type: ext4diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/deep_tree.yaml000066400000000000000000000005261324273057700304740ustar00rootroot00000000000000- local_loop: name: image0 - partitioning: name: mbr base: image0 label: mbr partitions: - name: root flags: [ boot, primary ] size: 100% mkfs: type: ext4 mount: mount_point: / fstab: options: "defaults" fsck-passno: 1 diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/duplicate_fs_labels.yaml000066400000000000000000000021601324273057700325200ustar00rootroot00000000000000- local_loop: name: image0 - partitioning: base: image0 name: mbr label: mbr partitions: - name: root base: image0 flags: [ boot, primary ] size: 55% - name: var base: image0 size: 40% - name: var_log base: image0 size: 5% - mkfs: base: root name: mkfs_root label: duplicate type: xfs - mount: base: mkfs_root name: mount_mkfs_root mount_point: / - fstab: base: mount_mkfs_root name: fstab_mount_mkfs_root fsck-passno: 1 options: defaults - mkfs: base: var name: mkfs_var label: duplicate type: xfs - mount: base: mkfs_var name: mount_mkfs_var mount_point: /var - fstab: base: mount_mkfs_var name: fstab_mount_mkfs_var fsck-passno: 1 options: defaults - mkfs: base: var_log name: mkfs_var_log type: xfs - mount: base: mkfs_var_log name: mount_mkfs_var_log mount_point: /var/log - fstab: base: mount_mkfs_var_log name: fstab_mount_mkfs_var_log fsck-passno: 1 options: defaults diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/duplicate_name.yaml000066400000000000000000000007321324273057700315110ustar00rootroot00000000000000- local_loop: name: this_is_a_duplicate - partitioning: base: this_is_a_duplicate name: root label: mbr partitions: - flags: [boot, primary] name: root base: image0 size: 100% - mount: base: mkfs_root name: this_is_a_duplicate mount_point: / - fstab: base: mount_mkfs_root name: fstab_mount_mkfs_root fsck-passno: 1 options: defaults - mkfs: base: root name: mkfs_root type: ext4 diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/lvm_graph.yaml000066400000000000000000000042011324273057700305110ustar00rootroot00000000000000- local_loop: name: image0 - partitioning: base: image0 name: mbr label: mbr partitions: - name: root base: image0 flags: [ boot,primary ] size: 3G - lvm: base: mbr name: lvm_mbr pvs: - name: pv options: ["--force"] base: root vgs: - name: vg base: ["pv"] options: ["--force"] lvs: - name: lv_root base: vg size: 1800M - name: lv_tmp base: vg size: 100M - name: lv_var base: vg size: 500M - name: lv_log base: vg size: 100M - name: lv_audit base: vg size: 100M - name: lv_home base: vg size: 200M - mkfs: name: fs_root base: lv_root label: "img-rootfs" type: "xfs" - mount: name: mount_fs_root base: fs_root mount_point: / - fstab: name: fstab_mount_fs_root base: mount_fs_root options: "rw,relatime" fsck-passno: 1 - mkfs: name: fs_var base: lv_var type: "xfs" - mount: name: mount_fs_var base: fs_var mount_point: /var - fstab: name: fstab_mount_fs_var base: mount_fs_var options: "rw,relatime" - mkfs: name: fs_log base: lv_log type: "xfs" - mount: name: mount_fs_log base: fs_log mount_point: /var/log - fstab: name: fstab_mount_fs_log base: mount_fs_log options: "rw,relatime" - mkfs: name: fs_audit base: lv_audit type: "xfs" - mount: name: mount_fs_audit base: fs_audit mount_point: /var/log/audit - fstab: name: fstab_mount_fs_audit base: mount_fs_audit options: "rw,relatime" - mkfs: name: fs_tmp base: lv_tmp type: "xfs" - mount: name: mount_fs_tmp base: fs_tmp mount_point: /tmp - fstab: name: fstab_mount_fs_tmp base: mount_fs_tmp options: "rw,nosuid,nodev,noexec,relatime" - mkfs: name: fs_home base: lv_home type: "xfs" - mount: name: mount_fs_home base: fs_home mount_point: /home - fstab: name: fstab_mount_fs_home base: mount_fs_home options: "rw,nodev,relatime" diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/lvm_tree.yaml000066400000000000000000000032051324273057700303520ustar00rootroot00000000000000- local_loop: name: image0 - partitioning: base: image0 name: mbr label: mbr partitions: - name: root base: image0 flags: [ boot,primary ] size: 3G - lvm: base: mbr pvs: - name: pv options: ["--force"] base: root vgs: - name: vg base: ["pv"] options: ["--force"] lvs: - name: lv_root base: vg size: 1800M - name: lv_tmp base: vg size: 100M - name: lv_var base: vg size: 500M - name: lv_log base: vg size: 100M - name: lv_audit base: vg size: 100M - name: lv_home base: vg size: 200M - mkfs: name: fs_root base: lv_root label: "img-rootfs" type: "xfs" mount: mount_point: / fstab: options: "rw,relatime" fsck-passno: 1 - mkfs: name: fs_var base: lv_var type: "xfs" mount: mount_point: /var fstab: options: "rw,relatime" - mkfs: name: fs_log base: lv_log type: "xfs" mount: mount_point: /var/log fstab: options: "rw,relatime" - mkfs: name: fs_audit base: lv_audit type: "xfs" mount: mount_point: /var/log/audit fstab: options: "rw,relatime" - mkfs: name: fs_tmp base: lv_tmp type: "xfs" mount: mount_point: /tmp fstab: options: "rw,nosuid,nodev,noexec,relatime" - mkfs: name: fs_home base: lv_home type: "xfs" mount: mount_point: /home fstab: options: "rw,nodev,relatime" diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/lvm_tree_multiple_pv.yaml000066400000000000000000000033521324273057700327750ustar00rootroot00000000000000- local_loop: name: image0 - partitioning: base: image0 label: mbr partitions: - name: root flags: [ boot,primary ] size: 3G - name: data flags: [ primary ] size: 1G - lvm: name: lvm pvs: - name: pv options: ["--force"] base: root - name: pv1 options: ["--force"] base: data vgs: - name: vg base: ["pv", "pv1"] options: ["--force"] lvs: - name: lv_root base: vg size: 1800M - name: lv_tmp base: vg size: 100M - name: lv_var base: vg size: 500M - name: lv_log base: vg size: 100M - name: lv_audit base: vg size: 100M - name: lv_home base: vg size: 200M - mkfs: name: fs_root base: lv_root label: "img-rootfs" type: "xfs" mount: mount_point: / fstab: options: "rw,relatime" fsck-passno: 1 - mkfs: name: fs_var base: lv_var type: "xfs" mount: mount_point: /var fstab: options: "rw,relatime" - mkfs: name: fs_log base: lv_log type: "xfs" mount: mount_point: /var/log fstab: options: "rw,relatime" - mkfs: name: fs_audit base: lv_audit type: "xfs" mount: mount_point: /var/log/audit fstab: options: "rw,relatime" - mkfs: name: fs_tmp base: lv_tmp type: "xfs" mount: mount_point: /tmp fstab: options: "rw,nosuid,nodev,noexec,relatime" - mkfs: name: fs_home base: lv_home type: "xfs" mount: mount_point: /home fstab: options: "rw,nodev,relatime" diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/lvm_tree_multiple_pv_vg.yaml000066400000000000000000000034601324273057700334710ustar00rootroot00000000000000- local_loop: name: image0 - partitioning: base: image0 label: mbr partitions: - name: root flags: [ boot,primary ] size: 3G - name: data flags: [ primary ] size: 1G - lvm: name: lvm pvs: - name: pv options: ["--force"] base: root - name: pv1 options: ["--force"] base: data vgs: - name: vg1 base: ["pv"] options: ["--force"] - name: vg2 base: ["pv1"] options: ["--force"] lvs: - name: lv_root base: vg1 size: 1800M - name: lv_tmp base: vg1 size: 100M - name: lv_var base: vg2 size: 500M - name: lv_log base: vg2 size: 100M - name: lv_audit base: vg2 size: 100M - name: lv_home base: vg2 size: 200M - mkfs: name: fs_root base: lv_root label: "img-rootfs" type: "xfs" mount: mount_point: / fstab: options: "rw,relatime" fsck-passno: 1 - mkfs: name: fs_var base: lv_var type: "xfs" mount: mount_point: /var fstab: options: "rw,relatime" - mkfs: name: fs_log base: lv_log type: "xfs" mount: mount_point: /var/log fstab: options: "rw,relatime" - mkfs: name: fs_audit base: lv_audit type: "xfs" mount: mount_point: /var/log/audit fstab: options: "rw,relatime" - mkfs: name: fs_tmp base: lv_tmp type: "xfs" mount: mount_point: /tmp fstab: options: "rw,nosuid,nodev,noexec,relatime" - mkfs: name: fs_home base: lv_home type: "xfs" mount: mount_point: /home fstab: options: "rw,nodev,relatime" lvm_tree_partition_ordering.yaml000066400000000000000000000017571324273057700342670ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config- local_loop: name: image0 size: 5G - partitioning: base: image0 label: mbr partitions: - name: boot flags: [ boot,primary ] size: 1G - name: root flags: [ primary ] size: 3G - lvm: name: lvm base: [root] pvs: - name: pv base: root vgs: - name: vg base: ["pv"] lvs: - name: lv_root base: vg size: 2000M - name: lv_var base: vg size: 500M - mkfs: base: boot type: ext3 mount: mount_point: /boot fstab: options: "nodev,nosuid" fsck-passno: 2 - mkfs: name: mkfs_root base: lv_root label: "img-rootfs" type: "ext4" mount: mount_point: / fstab: options: "noacl,errors=remount-ro" fsck-passno: 1 - mkfs: name: mkfs_var base: lv_var type: "ext4" mount: mount_point: /var fstab: options: "noacl" fsck-passno: 2 diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/lvm_tree_spanned_vg.yaml000066400000000000000000000023451324273057700325620ustar00rootroot00000000000000# testing config for a LVM data volume group that spans two partitions - local_loop: name: image0 - partitioning: base: image0 label: mbr partitions: - name: root flags: [ boot,primary ] size: 3G - name: data1 flags: [ primary ] size: 1G - name: data2 flags: [ primary ] size: 1G - lvm: name: lvm pvs: - name: pv_root options: ["--force"] base: root - name: pv_data1 options: ["--force"] base: data1 - name: pv_data2 options: ["--force"] base: data2 vgs: - name: vg_root base: ["pv_root"] options: ["--force"] - name: vg_data base: ["pv_data1", "pv_data2"] options: ["--force"] lvs: - name: lv_root base: vg_root size: 1800M - name: lv_data base: vg_data size: 2G - mkfs: name: fs_root base: lv_root label: "img-rootfs" type: "xfs" mount: mount_point: / fstab: options: "rw,relatime" fsck-passno: 1 - mkfs: name: fs_data base: lv_data type: "xfs" mount: mount_point: /opt fstab: options: "rw,relatime" diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/multi_key_node.yaml000066400000000000000000000002011324273057700315350ustar00rootroot00000000000000- mkfs: name: root_fs base: root_part type: xfs mount: name: mount_root_fs base: root_fs mount_point: /diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/multiple_partitions_graph.yaml000066400000000000000000000022121324273057700340220ustar00rootroot00000000000000- local_loop: name: image0 - partitioning: base: image0 name: mbr label: mbr partitions: - name: root base: image0 flags: [ boot, primary ] size: 55% - name: var base: image0 size: 40% - name: var_log base: image0 size: 5% - mkfs: base: root name: mkfs_root type: xfs uuid: root-uuid-1234 - mount: base: mkfs_root name: mount_mkfs_root mount_point: / - fstab: base: mount_mkfs_root name: fstab_mount_mkfs_root fsck-passno: 1 options: defaults - mkfs: base: var name: mkfs_var type: xfs uuid: var-uuid-1234 - mount: base: mkfs_var name: mount_mkfs_var mount_point: /var - fstab: base: mount_mkfs_var name: fstab_mount_mkfs_var fsck-passno: 1 options: defaults - mkfs: base: var_log name: mkfs_var_log type: vfat label: varlog - mount: base: mkfs_var_log name: mount_mkfs_var_log mount_point: /var/log - fstab: base: mount_mkfs_var_log name: fstab_mount_mkfs_var_log fsck-passno: 1 options: defaults diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/multiple_partitions_tree.yaml000066400000000000000000000016061324273057700336660ustar00rootroot00000000000000- local_loop: name: image0 - partitioning: base: image0 name: mbr label: mbr partitions: - name: root flags: [ boot, primary ] size: 55% mkfs: type: xfs uuid: root-uuid-1234 mount: mount_point: / fstab: options: "defaults" fsck-passno: 1 - name: var size: 40% mkfs: type: xfs uuid: var-uuid-1234 mount: mount_point: /var fstab: options: "defaults" fsck-passno: 1 - name: var_log size: 5% mkfs: type: vfat label: varlog mount: mount_point: /var/log fstab: options: "defaults" fsck-passno: 1 diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/rollback.yaml000066400000000000000000000010501324273057700303220ustar00rootroot00000000000000- test_a: name: test_node_a rollback_one_arg: down rollback_two_arg: you - test_b: base: test_node_a name: test_node_b rollback_one_arg: let rollback_two_arg: gonna - test_a: base: test_node_b name: test_node_aa rollback_one_arg: never rollback_two_arg: up - test_b: base: test_node_aa name: test_node_bb rollback_one_arg: you rollback_two_arg: give - test_a: base: test_node_bb name: test_node_aaa rollback_one_arg: gonna rollback_two_arg: never trigger_rollback: yesdiskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/simple_graph.yaml000066400000000000000000000002021324273057700312010ustar00rootroot00000000000000- mkfs: name: root_fs base: root_part type: xfs - mount: name: mount_root_fs base: root_fs mount_point: /diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/simple_tree.yaml000066400000000000000000000001331324273057700310420ustar00rootroot00000000000000- mkfs: name: root_fs base: root_part type: xfs mount: mount_point: /diskimage-builder-2.11.0/diskimage_builder/block_device/tests/config/too_long_fs_label.yaml000066400000000000000000000001541324273057700322040ustar00rootroot00000000000000- mkfs: base: fake name: mkfs_root label: this_label_is_too_long_to_work_with_xfs type: xfs diskimage-builder-2.11.0/diskimage_builder/block_device/tests/plugin/000077500000000000000000000000001324273057700257025ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/tests/plugin/__init__.py000066400000000000000000000000001324273057700300010ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/block_device/tests/plugin/test_a.py000066400000000000000000000053731324273057700275430ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # plugin test case import logging from diskimage_builder.block_device.plugin import NodeBase from diskimage_builder.block_device.plugin import PluginBase logger = logging.getLogger(__name__) class TestANode(NodeBase): def __init__(self, config, state, test_rollback): logger.debug("Create test 1") super(TestANode, self).__init__(config['name'], state) # might be a root node, so possibly no base if 'base' in config: self.base = config['base'] # put something in the state for test_b to check for state['test_init_state'] = 'here' # If we're doing rollback testing the config has some strings # set for us if test_rollback: self.add_rollback(self.do_rollback, config['rollback_one_arg']) self.add_rollback(self.do_rollback, config['rollback_two_arg']) # see if we're the node who is going to fail self.trigger_rollback = True if 'trigger_rollback' in config else False def get_edges(self): # may not have a base, if used as root node to = [self.base] if hasattr(self, 'base') else [] return (to, []) def do_rollback(self, string): # We will check this after all rollbacks to make sure they ran # in the right order self.state['rollback_test'].append(string) def create(self): # put some fake entries into state self.state['test_a'] = {} self.state['test_a']['value'] = 'foo' self.state['test_a']['value2'] = 'bar' if self.trigger_rollback: # The rollback test will append the strings to this as # it unrolls, and we'll check it's value at the end self.state['rollback_test'] = [] raise RuntimeError("Rollback triggered") return def umount(self): # Umount is run in reverse. This key should exist from test_b self.state['umount'].append('test_a') class TestA(PluginBase): def __init__(self, config, defaults, state): super(TestA, self).__init__() test_rollback = True if 'test_rollback' in defaults else False self.node = TestANode(config, state, test_rollback) def get_nodes(self): return [self.node] diskimage-builder-2.11.0/diskimage_builder/block_device/tests/plugin/test_b.py000066400000000000000000000044641324273057700275440ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # plugin test case import logging from diskimage_builder.block_device.plugin import NodeBase from diskimage_builder.block_device.plugin import PluginBase logger = logging.getLogger(__name__) class TestBNode(NodeBase): def __init__(self, config, state, test_rollback): logger.debug("Create test 1") super(TestBNode, self).__init__(config['name'], state) self.base = config['base'] # If we're doing rollback testing the config has some strings # set for us. if test_rollback: self.add_rollback(self.do_rollback, config['rollback_one_arg']) self.add_rollback(self.do_rollback, config['rollback_two_arg']) def get_edges(self): # this should have been inserted by test_a before # we are called assert self.state['test_init_state'] == 'here' return ([self.base], []) def do_rollback(self, string): # We will check this after all rollbacks to make sure they ran # in the right order self.state['rollback_test'].append(string) def create(self): self.state['test_b'] = {} self.state['test_b']['value'] = 'baz' return def umount(self): # these values should have persisteted from create() assert self.state['test_b']['value'] == 'baz' # umount run in reverse. this should run before test_a assert 'umount' not in self.state self.state['umount'] = [] self.state['umount'].append('test_b') class TestB(PluginBase): def __init__(self, config, defaults, state): super(TestB, self).__init__() test_rollback = True if 'test_rollback' in defaults else False self.node = TestBNode(config, state, test_rollback) def get_nodes(self): return [self.node] diskimage-builder-2.11.0/diskimage_builder/block_device/tests/test_base.py000066400000000000000000000025421324273057700267320ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import fixtures import logging import os import testtools import yaml logger = logging.getLogger(__name__) class TestBase(testtools.TestCase): """Base for all test cases""" def setUp(self): super(TestBase, self).setUp() fs = '%(asctime)s %(levelname)s [%(name)s] %(message)s' self.log_fixture = self.useFixture( fixtures.FakeLogger(level=logging.DEBUG, format=fs)) def get_config_file(self, f): """Get the full path to sample config file f """ logger.debug(os.path.dirname(__file__)) return os.path.join(os.path.dirname(__file__), 'config', f) def load_config_file(self, f): """Load f and return it after yaml parsing""" path = self.get_config_file(f) with open(path, 'r') as config: return yaml.safe_load(config) diskimage-builder-2.11.0/diskimage_builder/block_device/tests/test_config.py000066400000000000000000000137471324273057700272760ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import logging from diskimage_builder.block_device.config import config_tree_to_graph from diskimage_builder.block_device.config import create_graph from diskimage_builder.block_device.exception import \ BlockDeviceSetupException from diskimage_builder.block_device.tests.test_base import TestBase logger = logging.getLogger(__name__) class TestConfig(TestBase): """Helper for setting up and reading a config""" def setUp(self): super(TestConfig, self).setUp() # previously we mocked some globals here ... class TestGraphGeneration(TestConfig): """Extra helper class for testing graph generation""" def setUp(self): super(TestGraphGeneration, self).setUp() self.fake_default_config = { 'build-dir': '/fake', 'image-size': '1000', 'image-dir': '/fake', 'mount-base': '/fake', } class TestConfigParsing(TestConfig): """Test parsing config file into a graph""" # test an entry in the config not being a valid plugin def test_config_bad_plugin(self): config = self.load_config_file('bad_plugin.yaml') self.assertRaises(BlockDeviceSetupException, config_tree_to_graph, config) # test a config that has multiple keys for a top-level entry def test_config_multikey_node(self): config = self.load_config_file('multi_key_node.yaml') self.assertRaisesRegex(BlockDeviceSetupException, "Config entry top-level should be a single " "dict:", config_tree_to_graph, config) # a graph should remain the same def test_graph(self): graph = self.load_config_file('simple_graph.yaml') parsed_graph = config_tree_to_graph(graph) self.assertItemsEqual(parsed_graph, graph) # equivalence of simple tree to graph def test_simple_tree(self): tree = self.load_config_file('simple_tree.yaml') graph = self.load_config_file('simple_graph.yaml') parsed_graph = config_tree_to_graph(tree) self.assertItemsEqual(parsed_graph, graph) # equivalence of a deeper tree to graph def test_deep_tree(self): tree = self.load_config_file('deep_tree.yaml') graph = self.load_config_file('deep_graph.yaml') parsed_graph = config_tree_to_graph(tree) self.assertItemsEqual(parsed_graph, graph) # equivalence of a complicated multi-partition tree to graph def test_multipart_tree(self): tree = self.load_config_file('multiple_partitions_tree.yaml') graph = self.load_config_file('multiple_partitions_graph.yaml') parsed_graph = config_tree_to_graph(tree) logger.debug(parsed_graph) self.assertItemsEqual(parsed_graph, graph) class TestCreateGraph(TestGraphGeneration): # Test a graph with bad edge pointing to an invalid node def test_invalid_missing(self): config = self.load_config_file('bad_edge_graph.yaml') self.assertRaisesRegex(BlockDeviceSetupException, "Edge not defined: this_is_not_a_node", create_graph, config, self.fake_default_config, {}) # Test a graph with bad edge pointing to an invalid node def test_duplicate_name(self): config = self.load_config_file('duplicate_name.yaml') self.assertRaisesRegex(BlockDeviceSetupException, "Duplicate node name: " "this_is_a_duplicate", create_graph, config, self.fake_default_config, {}) # Test digraph generation from deep_graph config file def test_deep_graph_generator(self): config = self.load_config_file('deep_graph.yaml') graph, call_order = create_graph(config, self.fake_default_config, {}) call_order_list = [n.name for n in call_order] # manually created from deep_graph.yaml # Note unlike below, the sort here is stable because the graph # doesn't have multiple paths with only one partition call_order_names = ['image0', 'root', 'mkfs_root', 'mount_mkfs_root', 'fstab_mount_mkfs_root'] self.assertListEqual(call_order_list, call_order_names) # Test multiple partition digraph generation def test_multiple_partitions_graph_generator(self): config = self.load_config_file('multiple_partitions_graph.yaml') graph, call_order = create_graph(config, self.fake_default_config, {}) call_order_list = [n.name for n in call_order] # The sort creating call_order_list is unstable. # We want to ensure we see the "partitions" object in # root->var->var_log order root_pos = call_order_list.index('root') var_pos = call_order_list.index('var') var_log_pos = call_order_list.index('var_log') self.assertGreater(var_pos, root_pos) self.assertGreater(var_log_pos, var_pos) # Ensure mkfs happens after partition mkfs_root_pos = call_order_list.index('mkfs_root') self.assertLess(root_pos, mkfs_root_pos) mkfs_var_pos = call_order_list.index('mkfs_var') self.assertLess(var_pos, mkfs_var_pos) mkfs_var_log_pos = call_order_list.index('mkfs_var_log') self.assertLess(var_log_pos, mkfs_var_log_pos) diskimage-builder-2.11.0/diskimage_builder/block_device/tests/test_lvm.py000066400000000000000000000421521324273057700266170ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import copy import logging import mock import diskimage_builder.block_device.tests.test_config as tc from diskimage_builder.block_device.blockdevice import BlockDeviceState from diskimage_builder.block_device.config import config_tree_to_graph from diskimage_builder.block_device.config import create_graph from diskimage_builder.block_device.exception import \ BlockDeviceSetupException from diskimage_builder.block_device.level1.lvm import LVMCleanupNode from diskimage_builder.block_device.level1.lvm import LVMNode from diskimage_builder.block_device.level1.lvm import LVMPlugin from diskimage_builder.block_device.level1.lvm import LvsNode from diskimage_builder.block_device.level1.lvm import PvsNode from diskimage_builder.block_device.level1.lvm import VgsNode logger = logging.getLogger(__name__) class TestLVM(tc.TestGraphGeneration): def test_lvm_tree_to_graph(self): # equivalence of tree-based to graph-based config tree = self.load_config_file('lvm_tree.yaml') graph = self.load_config_file('lvm_graph.yaml') parsed_graph = config_tree_to_graph(tree) self.assertItemsEqual(parsed_graph, graph) def test_lvm_invalid_config(self): # test some invalid config paths config = self.load_config_file('lvm_graph.yaml') lvm_config = config[2]['lvm'] bad_config = copy.deepcopy(lvm_config) bad_config['vgs'][0]['base'] = ['invalid_pv'] self.assertRaisesRegex(BlockDeviceSetupException, "base:invalid_pv in vgs does not match " "a valid pvs", LVMPlugin, bad_config, {}, {}) bad_config = copy.deepcopy(lvm_config) bad_config['lvs'][0]['base'] = ['invalid_vg'] self.assertRaisesRegex(BlockDeviceSetupException, "base:\['invalid_vg'\] in lvs does not match " "a valid vg", LVMPlugin, bad_config, {}, {}) bad_config = copy.deepcopy(lvm_config) del(bad_config['lvs'][0]['size']) self.assertRaisesRegex(BlockDeviceSetupException, "Missing 'size' or 'extents' in lvs config", LVMPlugin, bad_config, {}, {}) @mock.patch('diskimage_builder.block_device.level1.lvm.exec_sudo') def test_lvm_multi_pv(self, mock_exec_sudo): # Test the command-sequence for a more complicated LVM setup tree = self.load_config_file('lvm_tree_multiple_pv.yaml') config = config_tree_to_graph(tree) state = BlockDeviceState() graph, call_order = create_graph(config, self.fake_default_config, state) # XXX: todo; test call_order. Make sure PV's come before, VG; # VG before LV, and that mounts/etc happen afterwards. # Fake state for the two PV's specified by this config state['blockdev'] = {} state['blockdev']['root'] = {} state['blockdev']['root']['device'] = '/dev/fake/root' state['blockdev']['data'] = {} state['blockdev']['data']['device'] = '/dev/fake/data' for node in call_order: # XXX: This has not mocked out the "lower" layers of # creating the devices, which we're assuming works OK, nor # the upper layers. if isinstance(node, (LVMNode, LVMCleanupNode, PvsNode, VgsNode, LvsNode)): # only the LVMNode actually does anything here... node.create() # ensure the sequence of calls correctly setup the devices cmd_sequence = [ # create the pv's on the faked out block devices mock.call(['pvcreate', '/dev/fake/root', '--force']), mock.call(['pvcreate', '/dev/fake/data', '--force']), # create a volume called "vg" out of these two pv's mock.call(['vgcreate', 'vg', '/dev/fake/root', '/dev/fake/data', '--force']), # create a bunch of lv's on vg mock.call(['lvcreate', '--name', 'lv_root', '-L', '1800M', 'vg']), mock.call(['lvcreate', '--name', 'lv_tmp', '-L', '100M', 'vg']), mock.call(['lvcreate', '--name', 'lv_var', '-L', '500M', 'vg']), mock.call(['lvcreate', '--name', 'lv_log', '-L', '100M', 'vg']), mock.call(['lvcreate', '--name', 'lv_audit', '-L', '100M', 'vg']), mock.call(['lvcreate', '--name', 'lv_home', '-L', '200M', 'vg'])] self.assertEqual(mock_exec_sudo.call_count, len(cmd_sequence)) mock_exec_sudo.assert_has_calls(cmd_sequence) # Ensure the correct LVM state was preserved blockdev_state = { 'data': {'device': '/dev/fake/data'}, 'root': {'device': '/dev/fake/root'}, 'lv_audit': { 'device': '/dev/mapper/vg-lv_audit', 'extents': None, 'opts': None, 'size': '100M', 'vgs': 'vg' }, 'lv_home': { 'device': '/dev/mapper/vg-lv_home', 'extents': None, 'opts': None, 'size': '200M', 'vgs': 'vg' }, 'lv_log': { 'device': '/dev/mapper/vg-lv_log', 'extents': None, 'opts': None, 'size': '100M', 'vgs': 'vg' }, 'lv_root': { 'device': '/dev/mapper/vg-lv_root', 'extents': None, 'opts': None, 'size': '1800M', 'vgs': 'vg' }, 'lv_tmp': { 'device': '/dev/mapper/vg-lv_tmp', 'extents': None, 'opts': None, 'size': '100M', 'vgs': 'vg' }, 'lv_var': { 'device': '/dev/mapper/vg-lv_var', 'extents': None, 'opts': None, 'size': '500M', 'vgs': 'vg' }, } # state.debug_dump() self.assertDictEqual(state['blockdev'], blockdev_state) # XXX: mount ordering? fs creation? def test_lvm_multi_pv_vg(self): # Test the command-sequence for a more complicated LVM setup tree = self.load_config_file('lvm_tree_multiple_pv_vg.yaml') config = config_tree_to_graph(tree) state = BlockDeviceState() graph, call_order = create_graph(config, self.fake_default_config, state) # XXX: todo; test call_order. Make sure PV's come before, VG; # VG before LV, and that mounts/etc happen afterwards. # Fake state for the two PV's specified by this config state['blockdev'] = {} state['blockdev']['root'] = {} state['blockdev']['root']['device'] = '/dev/fake/root' state['blockdev']['data'] = {} state['blockdev']['data']['device'] = '/dev/fake/data' # We mock patch this ... it's just a little long! exec_sudo = 'diskimage_builder.block_device.level1.lvm.exec_sudo' # # Creation test # with mock.patch(exec_sudo) as mock_exec_sudo: for node in call_order: # XXX: This has not mocked out the "lower" layers of # creating the devices, which we're assuming works OK, nor # the upper layers. if isinstance(node, (LVMNode, LVMCleanupNode, PvsNode, VgsNode, LvsNode)): # only the PvsNode actually does anything here... node.create() # ensure the sequence of calls correctly setup the devices cmd_sequence = [ # create the pv's on the faked out block devices mock.call(['pvcreate', '/dev/fake/root', '--force']), mock.call(['pvcreate', '/dev/fake/data', '--force']), # create a volume called "vg" out of these two pv's mock.call(['vgcreate', 'vg1', '/dev/fake/root', '--force']), mock.call(['vgcreate', 'vg2', '/dev/fake/data', '--force']), # create a bunch of lv's on vg mock.call(['lvcreate', '--name', 'lv_root', '-L', '1800M', 'vg1']), mock.call(['lvcreate', '--name', 'lv_tmp', '-L', '100M', 'vg1']), mock.call(['lvcreate', '--name', 'lv_var', '-L', '500M', 'vg2']), mock.call(['lvcreate', '--name', 'lv_log', '-L', '100M', 'vg2']), mock.call(['lvcreate', '--name', 'lv_audit', '-L', '100M', 'vg2']), mock.call(['lvcreate', '--name', 'lv_home', '-L', '200M', 'vg2'])] self.assertListEqual(mock_exec_sudo.call_args_list, cmd_sequence) # Ensure the correct LVM state was preserved blockdev_state = { 'data': {'device': '/dev/fake/data'}, 'root': {'device': '/dev/fake/root'}, 'lv_audit': { 'device': '/dev/mapper/vg2-lv_audit', 'extents': None, 'opts': None, 'size': '100M', 'vgs': 'vg2' }, 'lv_home': { 'device': '/dev/mapper/vg2-lv_home', 'extents': None, 'opts': None, 'size': '200M', 'vgs': 'vg2' }, 'lv_log': { 'device': '/dev/mapper/vg2-lv_log', 'extents': None, 'opts': None, 'size': '100M', 'vgs': 'vg2' }, 'lv_root': { 'device': '/dev/mapper/vg1-lv_root', 'extents': None, 'opts': None, 'size': '1800M', 'vgs': 'vg1' }, 'lv_tmp': { 'device': '/dev/mapper/vg1-lv_tmp', 'extents': None, 'opts': None, 'size': '100M', 'vgs': 'vg1' }, 'lv_var': { 'device': '/dev/mapper/vg2-lv_var', 'extents': None, 'opts': None, 'size': '500M', 'vgs': 'vg2' }, } # state.debug_dump() self.assertDictEqual(state['blockdev'], blockdev_state) # # Cleanup test # with mock.patch(exec_sudo) as mock_exec_sudo, \ mock.patch('tempfile.NamedTemporaryFile') as mock_temp, \ mock.patch('os.unlink'): # each call to tempfile.NamedTemporaryFile will return a # new mock with a unique filename, which we store in # tempfiles tempfiles = [] def new_tempfile(*args, **kwargs): n = '/tmp/files%s' % len(tempfiles) # trap! note mock.Mock(name = n) doesn't work like you # think it would, since mock has a name attribute. # That's why we override it with the configure_mock # (this is mentioned in mock documentation if you read # it :) r = mock.Mock() r.configure_mock(name=n) tempfiles.append(n) return r mock_temp.side_effect = new_tempfile reverse_order = reversed(call_order) for node in reverse_order: if isinstance(node, (LVMNode, LVMCleanupNode, PvsNode, VgsNode, LvsNode)): node.cleanup() cmd_sequence = [ # delete the lv's mock.call(['lvchange', '-an', '/dev/vg1/lv_root']), mock.call(['lvchange', '-an', '/dev/vg1/lv_tmp']), mock.call(['lvchange', '-an', '/dev/vg2/lv_var']), mock.call(['lvchange', '-an', '/dev/vg2/lv_log']), mock.call(['lvchange', '-an', '/dev/vg2/lv_audit']), mock.call(['lvchange', '-an', '/dev/vg2/lv_home']), # delete the vg's mock.call(['vgchange', '-an', 'vg1']), mock.call(['vgchange', '-an', 'vg2']), mock.call(['udevadm', 'settle']), mock.call(['pvscan', '--cache']), ] self.assertListEqual(mock_exec_sudo.call_args_list, cmd_sequence) def test_lvm_spanned_vg(self): # Test when a volume group spans some partitions tree = self.load_config_file('lvm_tree_spanned_vg.yaml') config = config_tree_to_graph(tree) state = BlockDeviceState() graph, call_order = create_graph(config, self.fake_default_config, state) # XXX: todo; test call_order. Make sure PV's come before, VG; # VG before LV, and that mounts/etc happen afterwards. # Fake state for the two PV's specified by this config state['blockdev'] = {} state['blockdev']['root'] = {} state['blockdev']['root']['device'] = '/dev/fake/root' state['blockdev']['data1'] = {} state['blockdev']['data1']['device'] = '/dev/fake/data1' state['blockdev']['data2'] = {} state['blockdev']['data2']['device'] = '/dev/fake/data2' # We mock patch this ... it's just a little long! exec_sudo = 'diskimage_builder.block_device.level1.lvm.exec_sudo' # # Creation test # with mock.patch(exec_sudo) as mock_exec_sudo: for node in call_order: # XXX: This has not mocked out the "lower" layers of # creating the devices, which we're assuming works OK, nor # the upper layers. if isinstance(node, (LVMNode, LVMCleanupNode, PvsNode, VgsNode, LvsNode)): # only the LVMNode actually does anything here... node.create() # ensure the sequence of calls correctly setup the devices cmd_sequence = [ # create the pv's on the faked out block devices mock.call(['pvcreate', '/dev/fake/root', '--force']), mock.call(['pvcreate', '/dev/fake/data1', '--force']), mock.call(['pvcreate', '/dev/fake/data2', '--force']), # create a root and a data volume, with the data volume # spanning data1 & data2 mock.call(['vgcreate', 'vg_root', '/dev/fake/root', '--force']), mock.call(['vgcreate', 'vg_data', '/dev/fake/data1', '/dev/fake/data2', '--force']), # create root and data volume mock.call(['lvcreate', '--name', 'lv_root', '-L', '1800M', 'vg_root']), mock.call(['lvcreate', '--name', 'lv_data', '-L', '2G', 'vg_data']) ] self.assertListEqual(mock_exec_sudo.call_args_list, cmd_sequence) with mock.patch(exec_sudo) as mock_exec_sudo, \ mock.patch('tempfile.NamedTemporaryFile') as mock_temp, \ mock.patch('os.unlink'): # see above ... tempfiles = [] def new_tempfile(*args, **kwargs): n = '/tmp/files%s' % len(tempfiles) r = mock.Mock() r.configure_mock(name=n) tempfiles.append(n) return r mock_temp.side_effect = new_tempfile reverse_order = reversed(call_order) for node in reverse_order: if isinstance(node, (LVMNode, PvsNode, VgsNode, LvsNode)): node.cleanup() cmd_sequence = [ # deactivate lv's mock.call(['lvchange', '-an', '/dev/vg_root/lv_root']), mock.call(['lvchange', '-an', '/dev/vg_data/lv_data']), # deactivate vg's mock.call(['vgchange', '-an', 'vg_root']), mock.call(['vgchange', '-an', 'vg_data']), mock.call(['udevadm', 'settle']), ] self.assertListEqual(mock_exec_sudo.call_args_list, cmd_sequence) diskimage-builder-2.11.0/diskimage_builder/block_device/tests/test_mbr.py000066400000000000000000000147521324273057700266060ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import fixtures import logging import mock import os import subprocess import diskimage_builder.block_device.tests.test_base as tb from diskimage_builder.block_device.level0.localloop import image_create from diskimage_builder.block_device.level1.mbr import MBR logger = logging.getLogger(__name__) class TestMBR(tb.TestBase): disk_size_10M = 10 * 1024 * 1024 disk_size_1G = 1024 * 1024 * 1024 def _get_path_for_partx(self): """Searches and sets the path for partx Because different distributions store the partx binary at different places, there is the need to look for it. """ dirs = ["/bin", "/usr/bin", "/sbin", "/usr/sbin"] for d in dirs: if os.path.exists(os.path.join(d, "partx")): return os.path.join(d, "partx") return # If not found, try without path. return "partx" def setUp(self): super(TestMBR, self).setUp() self.tmp_dir = fixtures.TempDir() self.useFixture(self.tmp_dir) self.image_path = os.path.join(self.tmp_dir.path, "image.raw") image_create(self.image_path, TestMBR.disk_size_1G) logger.debug("Temp image is %s", self.image_path) self.partx_args = [self._get_path_for_partx(), "--raw", "--output", "NR,START,END,TYPE,FLAGS,SCHEME", "-g", "-b", "-", self.image_path] def _run_partx(self, image_path): logger.info("Running command: %s", self.partx_args) return subprocess.check_output(self.partx_args).decode("ascii") @mock.patch('os.fsync', wraps=os.fsync) def test_one_ext_partition(self, mock_os_fsync): """Creates one partition and check correctness with partx.""" with MBR(self.image_path, TestMBR.disk_size_1G, 1024 * 1024) as mbr: mbr.add_partition(False, False, TestMBR.disk_size_10M, 0x83) # the exit handler of MBR should have synced the raw device # before exit mock_os_fsync.assert_called() output = self._run_partx(self.image_path) self.assertEqual( "1 2048 2097151 0xf 0x0 dos\n" "5 4096 24575 0x83 0x0 dos\n", output) def test_zero_partitions(self): """Creates no partition and check correctness with partx.""" with MBR(self.image_path, TestMBR.disk_size_1G, 1024 * 1024): pass output = self._run_partx(self.image_path) self.assertEqual("", output) def test_many_ext_partitions(self): """Creates many partition and check correctness with partx.""" with MBR(self.image_path, TestMBR.disk_size_1G, 1024 * 1024) as mbr: for nr in range(0, 64): mbr.add_partition(False, False, TestMBR.disk_size_10M, 0x83) output = self._run_partx(self.image_path) lines = output.split("\n") self.assertEqual(66, len(lines)) self.assertEqual( "1 2048 2097151 0xf 0x0 dos", lines[0]) start_block = 4096 end_block = start_block + TestMBR.disk_size_10M / 512 - 1 for nr in range(1, 65): fields = lines[nr].split(" ") self.assertEqual(6, len(fields)) self.assertEqual(nr + 4, int(fields[0])) self.assertEqual(start_block, int(fields[1])) self.assertEqual(end_block, int(fields[2])) self.assertEqual("0x83", fields[3]) self.assertEqual("0x0", fields[4]) self.assertEqual("dos", fields[5]) start_block += 22528 end_block = start_block + TestMBR.disk_size_10M / 512 - 1 def test_one_pri_partition(self): """Creates one primary partition and check correctness with partx.""" with MBR(self.image_path, TestMBR.disk_size_1G, 1024 * 1024) as mbr: mbr.add_partition(True, False, TestMBR.disk_size_10M, 0x83) output = self._run_partx(self.image_path) self.assertEqual( "1 2048 22527 0x83 0x0 dos\n", output) def test_three_pri_partition(self): """Creates three primary partition and check correctness with partx.""" with MBR(self.image_path, TestMBR.disk_size_1G, 1024 * 1024) as mbr: for _ in range(3): mbr.add_partition(True, False, TestMBR.disk_size_10M, 0x83) output = self._run_partx(self.image_path) self.assertEqual( "1 2048 22527 0x83 0x0 dos\n" "2 22528 43007 0x83 0x0 dos\n" "3 43008 63487 0x83 0x0 dos\n", output) def test_many_pri_and_ext_partition(self): """Creates many primary and extended partitions.""" with MBR(self.image_path, TestMBR.disk_size_1G, 1024 * 1024) as mbr: # Create three primary partitions for _ in range(3): mbr.add_partition(True, False, TestMBR.disk_size_10M, 0x83) for _ in range(7): mbr.add_partition(False, False, TestMBR.disk_size_10M, 0x83) output = self._run_partx(self.image_path) self.assertEqual( "1 2048 22527 0x83 0x0 dos\n" # Primary 1 "2 22528 43007 0x83 0x0 dos\n" # Primary 2 "3 43008 63487 0x83 0x0 dos\n" # Primary 3 "4 63488 2097151 0xf 0x0 dos\n" # Extended "5 65536 86015 0x83 0x0 dos\n" # Extended Partition 1 "6 88064 108543 0x83 0x0 dos\n" # Extended Partition 2 "7 110592 131071 0x83 0x0 dos\n" # ... "8 133120 153599 0x83 0x0 dos\n" "9 155648 176127 0x83 0x0 dos\n" "10 178176 198655 0x83 0x0 dos\n" "11 200704 221183 0x83 0x0 dos\n", output) def test_pri_fat32_lba_partition(self): """Creates a partition with a non-default 'type' and verifies.""" with MBR(self.image_path, TestMBR.disk_size_1G, 1024 * 1024) as mbr: mbr.add_partition(True, False, TestMBR.disk_size_10M, 0x0c) output = self._run_partx(self.image_path) self.assertEqual( "1 2048 22527 0xc 0x0 dos\n", output) diskimage-builder-2.11.0/diskimage_builder/block_device/tests/test_mkfs.py000066400000000000000000000027601324273057700267620ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import logging import diskimage_builder.block_device.tests.test_config as tc from diskimage_builder.block_device.config import create_graph from diskimage_builder.block_device.exception import \ BlockDeviceSetupException logger = logging.getLogger(__name__) class TestMkfs(tc.TestGraphGeneration): def test_duplicate_labels(self): config = self.load_config_file('duplicate_fs_labels.yaml') self.assertRaisesRegex(BlockDeviceSetupException, "used more than once", create_graph, config, self.fake_default_config, {}) def test_too_long_labels(self): config = self.load_config_file('too_long_fs_label.yaml') self.assertRaisesRegex(BlockDeviceSetupException, "too long for filesystem", create_graph, config, self.fake_default_config, {}) diskimage-builder-2.11.0/diskimage_builder/block_device/tests/test_mount_order.py000066400000000000000000000171421324273057700303570ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import functools import logging import mock import os import diskimage_builder.block_device.tests.test_config as tc from diskimage_builder.block_device.config import config_tree_to_graph from diskimage_builder.block_device.config import create_graph from diskimage_builder.block_device.level2.mkfs import FilesystemNode from diskimage_builder.block_device.level3.mount import cmp_mount_order from diskimage_builder.block_device.level3.mount import MountPointNode from diskimage_builder.block_device.tests.test_base import TestBase logger = logging.getLogger(__name__) class TestMountComparator(TestBase): def test_mount_comparator(self): # This tests cmp_mount_order to ensure it sorts in the # expected order. The comparator takes a tuple of # (mount_point, node_name) but we can ignore the name partitions = [ ('/var/log', 'fake_log'), ('/boot', 'fake_boot'), ('/', 'fake_name'), ('/var', 'fake_name')] partitions.sort(key=functools.cmp_to_key(cmp_mount_order)) res = list(x[0] for x in partitions) # "/" must be first self.assertEqual(res[0], '/') # /var before /var/log var_pos = res.index('/var') var_log_pos = res.index('/var/log') self.assertGreater(var_log_pos, var_pos) class TestMountOrder(tc.TestGraphGeneration): def _exec_sudo_log(*args, **kwargs): # Used as a side-effect from exec_sudo mocking so we can see # the call in-place in logs logger.debug("exec_sudo: %s", " ".join(args[0])) @mock.patch('diskimage_builder.block_device.level3.mount.exec_sudo', side_effect=_exec_sudo_log) @mock.patch('diskimage_builder.block_device.level2.mkfs.exec_sudo', side_effect=_exec_sudo_log) def test_mfks_and_mount_order(self, mock_exec_sudo_mkfs, mock_exec_sudo_mount): # XXX: better mocking for the os.path.exists calls to avoid # failing if this exists. self.assertFalse(os.path.exists('/fake/')) # This is probably in order after graph creation, so ensure it # remains stable. We test the mount and umount call sequences config = self.load_config_file('multiple_partitions_graph.yaml') state = {} graph, call_order = create_graph(config, self.fake_default_config, state) # Mocked block device state state['blockdev'] = {} state['blockdev']['root'] = {'device': '/dev/loopXp1/root'} state['blockdev']['var'] = {'device': '/dev/loopXp2/var'} state['blockdev']['var_log'] = {'device': '/dev/loopXp3/var_log'} for node in call_order: if isinstance(node, (FilesystemNode, MountPointNode)): node.create() for node in reversed(call_order): if isinstance(node, (FilesystemNode, MountPointNode)): node.umount() # ensure that partitions were mounted in order root->var->var/log self.assertListEqual(state['mount_order'], ['/', '/var', '/var/log']) # fs creation sequence (note we don't care about order of this # as they're all independent) cmd_sequence = [ mock.call(['mkfs', '-t', 'xfs', '-L', 'mkfs_root', '-m', 'uuid=root-uuid-1234', '-q', '/dev/loopXp1/root']), mock.call(['mkfs', '-t', 'xfs', '-L', 'mkfs_var', '-m', 'uuid=var-uuid-1234', '-q', '/dev/loopXp2/var']), mock.call(['mkfs', '-t', 'vfat', '-n', 'varlog', '/dev/loopXp3/var_log']) ] self.assertEqual(mock_exec_sudo_mkfs.call_count, len(cmd_sequence)) mock_exec_sudo_mkfs.assert_has_calls(cmd_sequence, any_order=True) # Check mount sequence cmd_sequence = [ # mount sequence mock.call(['mkdir', '-p', '/fake/']), mock.call(['mount', '/dev/loopXp1/root', '/fake/']), mock.call(['mkdir', '-p', '/fake/var']), mock.call(['mount', '/dev/loopXp2/var', '/fake/var']), mock.call(['mkdir', '-p', '/fake/var/log']), mock.call(['mount', '/dev/loopXp3/var_log', '/fake/var/log']), # umount sequence mock.call(['sync']), # note /fake/var/log is a vfs partition to make sure # we don't try to fstrim it mock.call(['umount', '/fake/var/log']), mock.call(['sync']), mock.call(['fstrim', '--verbose', '/fake/var']), mock.call(['umount', '/fake/var']), mock.call(['sync']), mock.call(['fstrim', '--verbose', '/fake/']), mock.call(['umount', '/fake/']) ] self.assertListEqual(mock_exec_sudo_mount.call_args_list, cmd_sequence) @mock.patch('diskimage_builder.block_device.level3.mount.exec_sudo', side_effect=_exec_sudo_log) def test_mount_order_unsorted(self, mock_exec_sudo): # As above, but this is out of order and gets sorted # so that root is mounted first (and skips the mkfs testing). config = self.load_config_file('lvm_tree_partition_ordering.yaml') parsed_graph = config_tree_to_graph(config) state = {} graph, call_order = create_graph(parsed_graph, self.fake_default_config, state) state['filesys'] = { 'mkfs_root': { 'device': '/dev/loopXp1', 'fstype': 'xfs' }, 'mkfs_var': { 'device': '/dev/loopXp2', 'fstype': 'xfs', }, 'mkfs_boot': { 'device': '/dev/loopXp3', 'fstype': 'vfat', }, } for node in call_order: if isinstance(node, MountPointNode): node.create() for node in reversed(call_order): if isinstance(node, MountPointNode): node.umount() # ensure that partitions are mounted in order / -> /boot -> /var self.assertListEqual(state['mount_order'], ['/', '/boot', '/var']) cmd_sequence = [ # mount sequence mock.call(['mkdir', '-p', '/fake/']), mock.call(['mount', '/dev/loopXp1', '/fake/']), mock.call(['mkdir', '-p', '/fake/boot']), mock.call(['mount', '/dev/loopXp3', '/fake/boot']), mock.call(['mkdir', '-p', '/fake/var']), mock.call(['mount', '/dev/loopXp2', '/fake/var']), # umount sequence mock.call(['sync']), mock.call(['fstrim', '--verbose', '/fake/var']), mock.call(['umount', '/fake/var']), mock.call(['sync']), # no trim on vfat /fake/boot mock.call(['umount', '/fake/boot']), mock.call(['sync']), mock.call(['fstrim', '--verbose', '/fake/']), mock.call(['umount', '/fake/']) ] self.assertListEqual(mock_exec_sudo.call_args_list, cmd_sequence) diskimage-builder-2.11.0/diskimage_builder/block_device/tests/test_state.py000066400000000000000000000126271324273057700271450ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import codecs import fixtures import json import logging import os from stevedore import extension from testtools.matchers import FileExists import diskimage_builder.block_device.blockdevice as bd import diskimage_builder.block_device.tests.test_base as tb from diskimage_builder.block_device.exception import \ BlockDeviceSetupException logger = logging.getLogger(__name__) class TestStateBase(tb.TestBase): def setUp(self): super(TestStateBase, self).setUp() # override the extensions to the test extensions test_extensions = extension.ExtensionManager( namespace='diskimage_builder.block_device.plugin_test', invoke_on_load=False) extensions_fixture = fixtures.MonkeyPatch( 'diskimage_builder.block_device.config._extensions', test_extensions) self.useFixture(extensions_fixture) # status and other bits saved here self.build_dir = fixtures.TempDir() self.useFixture(self.build_dir) class TestState(TestStateBase): # The the state generation & saving methods def test_state_create(self): params = { 'build-dir': self.build_dir.path, 'config': self.get_config_file('cmd_create.yaml') } bd_obj = bd.BlockDevice(params) bd_obj.cmd_init() bd_obj.cmd_create() # cmd_create should have persisted this to disk state_file = bd_obj.state_json_file_name self.assertThat(state_file, FileExists()) # ensure we see the values put in by the test extensions # persisted with codecs.open(state_file, encoding='utf-8', mode='r') as fd: state = json.load(fd) self.assertDictEqual(state, {'test_a': {'value': 'foo', 'value2': 'bar'}, 'test_b': {'value': 'baz'}, 'test_init_state': 'here'}) pickle_file = bd_obj.node_pickle_file_name self.assertThat(pickle_file, FileExists()) # run umount, which should load the picked nodes and run in # reverse. This will create some state in "test_b" that it # added to by "test_a" ... ensuring it was run backwards. It # also checks the state was persisted through the pickling # process. bd_obj.cmd_umount() # Test state going missing between phases def test_missing_state(self): params = { 'build-dir': self.build_dir.path, 'config': self.get_config_file('cmd_create.yaml') } bd_obj = bd.BlockDevice(params) bd_obj.cmd_init() bd_obj.cmd_create() # cmd_create should have persisted this to disk state_file = bd_obj.state_json_file_name self.assertThat(state_file, FileExists()) pickle_file = bd_obj.node_pickle_file_name self.assertThat(pickle_file, FileExists()) # simulate the state somehow going missing, and ensure that # later calls notice os.unlink(state_file) os.unlink(pickle_file) # This reads from the state dump json file self.assertRaisesRegex(BlockDeviceSetupException, "State dump not found", bd_obj.cmd_getval, 'image-path') self.assertRaisesRegex(BlockDeviceSetupException, "State dump not found", bd_obj.cmd_writefstab) # this uses the pickled nodes self.assertRaisesRegex(BlockDeviceSetupException, "Pickle file not found", bd_obj.cmd_delete) self.assertRaisesRegex(BlockDeviceSetupException, "Pickle file not found", bd_obj.cmd_cleanup) # XXX: figure out unit test for umount # Test ordering of rollback calls if create() fails def test_rollback(self): params = { 'build-dir': self.build_dir.path, 'config': self.get_config_file('rollback.yaml'), 'test_rollback': True } bd_obj = bd.BlockDevice(params) bd_obj.cmd_init() # The config file has flags in that tell the last node to # fail, which will trigger the rollback. self.assertRaises(RuntimeError, bd_obj.cmd_create) # cmd_create should have persisted this to disk even after the # failure state_file = bd_obj.state_json_file_name self.assertThat(state_file, FileExists()) with codecs.open(state_file, encoding='utf-8', mode='r') as fd: state = json.load(fd) # ensure the rollback was called in order self.assertListEqual(state['rollback_test'], ['never', 'gonna', 'give', 'you', 'up', 'never', 'gonna', 'let', 'you', 'down']) diskimage-builder-2.11.0/diskimage_builder/block_device/tests/test_utils.py000066400000000000000000000042471324273057700271640ustar00rootroot00000000000000# Copyright 2016 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import logging import diskimage_builder.block_device.tests.test_base as tb from diskimage_builder.block_device.utils import parse_abs_size_spec from diskimage_builder.block_device.utils import parse_rel_size_spec logger = logging.getLogger(__name__) class TestBlockDeviceUtils(tb.TestBase): """Tests for the utils.py This tests mostly the error and failure cases - because the good cases are tested implicitly with the higher level unit tests. """ def test_parse_rel_size_with_abs(self): """Calls parse_rel_size_spec with an absolute number""" is_rel, size = parse_rel_size_spec("154MiB", 0) self.assertFalse(is_rel) self.assertEqual(154 * 1024 * 1024, size) def test_parse_abs_size_without_spec(self): """Call parse_abs_size_spec without spec""" size = parse_abs_size_spec("198") self.assertEqual(198, size) def test_invalid_unit_spec(self): """Call parse_abs_size_spec with invalid unit spec""" self.assertRaises(RuntimeError, parse_abs_size_spec, "747InVaLiDUnIt") def test_broken_unit_spec(self): """Call parse_abs_size_spec with a completely broken unit spec""" self.assertRaises(RuntimeError, parse_abs_size_spec, "_+!HuHi+-=") def test_parse_size_spec(self): map(lambda tspec: self.assertEqual(parse_abs_size_spec(tspec[0]), tspec[1]), [["20TiB", 20 * 1024**4], ["1024KiB", 1024 * 1024], ["1.2TB", 1.2 * 1000**4], ["2.4T", 2.4 * 1000**4], ["512B", 512], ["364", 364]]) diskimage-builder-2.11.0/diskimage_builder/block_device/utils.py000066400000000000000000000075361324273057700247670ustar00rootroot00000000000000# Copyright 2016 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import logging import re import subprocess logger = logging.getLogger(__name__) SIZE_UNIT_SPECS = [ ["TiB", 1024**4], ["GiB", 1024**3], ["MiB", 1024**2], ["KiB", 1024**1], ["TB", 1000**4], ["GB", 1000**3], ["MB", 1000**2], ["KB", 1000**1], ["T", 1000**4], ["G", 1000**3], ["M", 1000**2], ["K", 1000**1], ["B", 1], ["", 1], # No unit -> size is given in bytes ] # Basic RE to check and split floats (without exponent) # and a given unit specification (which must be non-numerical). size_unit_spec_re = re.compile("^([\d\.]*) ?([a-zA-Z0-9_]*)$") def _split_size_unit_spec(size_unit_spec): """Helper function to split unit specification into parts. The first part is the numeric part - the second one is the unit. """ match = size_unit_spec_re.match(size_unit_spec) if match is None: raise RuntimeError("Invalid size unit spec [%s]" % size_unit_spec) return match.group(1), match.group(2) def _get_unit_factor(unit_str): """Helper function to get the unit factor. The given unit_str needs to be a string of the SIZE_UNIT_SPECS table. If the unit is not found, a runtime error is raised. """ for spec_key, spec_value in SIZE_UNIT_SPECS: if unit_str == spec_key: return spec_value raise RuntimeError("unit_str [%s] not known" % unit_str) def parse_abs_size_spec(size_spec): size_cnt_str, size_unit_str = _split_size_unit_spec(size_spec) unit_factor = _get_unit_factor(size_unit_str) return int(unit_factor * ( float(size_cnt_str) if len(size_cnt_str) > 0 else 1)) def parse_rel_size_spec(size_spec, abs_size): """Parses size specifications - can be relative like 50% In addition to the absolute parsing also a relative parsing is done. If the size specification ends in '%', then the relative size of the given 'abs_size' is returned. """ if size_spec[-1] == '%': percent = float(size_spec[:-1]) return True, int(abs_size * percent / 100.0) return False, parse_abs_size_spec(size_spec) def exec_sudo(cmd): """Run a command under sudo Run command under sudo, with debug trace of output. This is like subprocess.check_call() but sudo wrapped and with output tracing at debug levels. Arguments: :param cmd: str command list; for Popen() :return: nothing :raises: subprocess.CalledProcessError if return code != 0 """ assert isinstance(cmd, list) sudo_cmd = ["sudo"] sudo_cmd.extend(cmd) try: logger.info("Calling [%s]", " ".join(sudo_cmd)) except TypeError: # Popen actually doesn't care, but we've managed to get mixed # str and bytes in argument lists which causes errors logging # commands. Give a clue as to what's going on. logger.exception("Ensure all arguments are str type!") raise proc = subprocess.Popen(sudo_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for line in iter(proc.stdout.readline, b""): logger.debug("exec_sudo: %s", line.rstrip()) proc.wait() if proc.returncode != 0: raise subprocess.CalledProcessError(proc.returncode, ' '.join(sudo_cmd)) diskimage-builder-2.11.0/diskimage_builder/disk_image_create.py000066400000000000000000000053271324273057700246310ustar00rootroot00000000000000# Copyright 2016 Ian Wienand (iwienand@redhat.com) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import os import os.path import runpy import sys import diskimage_builder.paths # borrowed from pip:locations.py def running_under_virtualenv(): """Return True if we're running inside a virtualenv, False otherwise.""" if hasattr(sys, 'real_prefix'): return True elif sys.prefix != getattr(sys, "base_prefix", sys.prefix): return True return False def activate_venv(): if running_under_virtualenv(): activate_this = os.path.join(sys.prefix, "bin", "activate_this.py") globs = runpy.run_path(activate_this, globals()) globals().update(globs) del globs def main(): # If we are called directly from a venv install # (/path/venv/bin/disk-image-create) then nothing has added the # virtualenv bin/ dir to $PATH. the exec'd script below will be # unable to find call other dib tools like dib-run-parts. # # One solution is to say that you should only ever run # disk-image-create in a shell that has already sourced # bin/activate.sh (all this really does is add /path/venv/bin to # $PATH). That's not a great interface as resulting errors will # be very non-obvious. # # We can detect if we are running in a virtualenv and use # virtualenv's "activate_this.py" script to activate it ourselves # before we call the script. This ensures we have the path setting activate_venv() environ = os.environ # pre-seed some paths for the shell script environ['_LIB'] = diskimage_builder.paths.get_path('lib') # export the path to the current python if not os.environ.get('DIB_PYTHON_EXEC'): os.environ['DIB_PYTHON_EXEC'] = sys.executable # we have to handle being called as "disk-image-create" or # "ramdisk-image-create". ramdisk-iamge-create is just a symlink # to disk-image-create # XXX: we could simplify things by removing the symlink, and # just setting IS_RAMDISK in environ here depending on sys.argv[1] script = "%s/%s" % (diskimage_builder.paths.get_path('lib'), os.path.basename(sys.argv[0])) os.execve("/bin/bash", ['bash', script] + sys.argv[1:], environ) diskimage-builder-2.11.0/diskimage_builder/element_dependencies.py000066400000000000000000000267041324273057700253530ustar00rootroot00000000000000# Copyright 2013 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from __future__ import print_function import argparse import collections import errno import logging import os import sys import yaml import diskimage_builder.logging_config logger = logging.getLogger(__name__) class MissingElementException(Exception): pass class AlreadyProvidedException(Exception): pass class MissingOSException(Exception): pass class InvalidElementDir(Exception): pass class Element(object): """An element""" def _get_element_set(self, path): """Get element set from element-[deps|provides] file Arguments: :param path: path to element description :return: the set of elements in the file, or a blank set if the file is not found. """ try: with open(path) as f: lines = (line.strip() for line in f) # Strip blanks, but do we want to strip comment lines # too? No use case at the moment, and comments might # break other things that poke at the element-* files. lines = (line for line in lines if line) return set(lines) except IOError as e: if e.errno == errno.ENOENT: return set([]) else: raise def _make_rdeps(self, all_elements): """Make a list of reverse dependencies (who depends on us). Only valid after _find_all_elements() Arguments: :param all_elements: dict as returned by _find_all_elements() :return: nothing, but elements will have r_depends var """ # note; deliberatly left out of __init__ so that accidental # access without init raises error self.r_depends = [] for name, element in all_elements.items(): if self.name in element.depends: self.r_depends.append(element.name) def __init__(self, name, path): """A new element :param name: The element name :param path: Full path to element. element-deps and element-provides files will be parsed """ self.name = name self.path = path # read the provides & depends files for this element into a # set; if the element has them. self.provides = self._get_element_set( os.path.join(path, 'element-provides')) self.depends = self._get_element_set( os.path.join(path, 'element-deps')) logger.debug("New element : %s", str(self)) def __eq__(self, other): return self.name == other.name def __repr__(self): return self.name def __str__(self): return '%s p:<%s> d:<%s>' % (self.name, ','.join(self.provides), ','.join(self.depends)) def _get_elements_dir(): if not os.environ.get('ELEMENTS_PATH'): raise Exception("$ELEMENTS_PATH must be set.") return os.environ['ELEMENTS_PATH'] def _expand_element_dependencies(user_elements, all_elements): """Expand user requested elements using element-deps files. Arguments: :param user_elements: iterable enumerating the elements a user requested :param all_elements: Element object dictionary from find_all_elements :return: a set containing the names of user_elements and all dependent elements including any transitive dependencies. """ final_elements = set(user_elements) check_queue = collections.deque(user_elements) provided = set() provided_by = collections.defaultdict(list) while check_queue: # bug #1303911 - run through the provided elements first to avoid # adding unwanted dependencies and looking for virtual elements element = check_queue.popleft() if element in provided: continue elif element not in all_elements: raise MissingElementException("Element '%s' not found" % element) element_obj = all_elements[element] element_deps = element_obj.depends element_provides = element_obj.provides # save which elements provide another element for potential # error message for provide in element_provides: provided_by[provide].append(element) provided.update(element_provides) check_queue.extend(element_deps - (final_elements | provided)) final_elements.update(element_deps) conflicts = set(user_elements) & provided if conflicts: logger.error( "The following elements are already provided by another element") for element in conflicts: logger.error("%s : already provided by %s", element, provided_by[element]) raise AlreadyProvidedException() if "operating-system" not in provided: raise MissingOSException("Please include an operating system element") out = final_elements - provided return([all_elements[element] for element in out]) def _find_all_elements(paths=None): """Build a dictionary Element() objects Walk ELEMENTS_PATH and find all elements. Make an Element object for each element we wish to consider. Note we process overrides such that elements specified earlier in the ELEMENTS_PATH override those seen later. :param paths: A list of paths to find elements in. If None will use ELEMENTS_PATH from environment :return: a dictionary of all elements """ all_elements = {} # note we process the later entries *first*, so that earlier # entries will override later ones. i.e. with # ELEMENTS_PATH=path1:path2:path3 # we want the elements in "path1" to override "path3" if not paths: paths = list(reversed(_get_elements_dir().split(':'))) else: paths = list(reversed(paths.split(':'))) logger.debug("ELEMENTS_PATH is: %s", ":".join(paths)) for path in paths: if not os.path.isdir(path): raise InvalidElementDir("ELEMENTS_PATH entry '%s' " "is not a directory " % path) # In words : make a list of directories in "path". Since an # element is a directory, this is our list of elements. elements = [os.path.realpath(os.path.join(path, f)) for f in os.listdir(path) if os.path.isdir(os.path.join(path, f))] for element in elements: # the element name is the last part of the full path in # element (these are all directories, we know that from # above) name = os.path.basename(element) new_element = Element(name, element) if name in all_elements: logger.warning("Element <%s> overrides <%s>", new_element.path, all_elements[name].path) all_elements[name] = new_element # Now we have all the elements, make a call on each element to # store it's reverse dependencies for name, element in all_elements.items(): element._make_rdeps(all_elements) return all_elements def _get_elements(elements, paths=None): """Return the canonical list of Element objects This function returns Element objects. For exernal calls, use get_elements which returns a simple tuple & list. :param elements: user specified list of elements :param paths: element paths, default to environment """ all_elements = _find_all_elements(paths) return _expand_element_dependencies(elements, all_elements) def get_elements(elements, paths=None): """Return the canonical list of elements with their dependencies .. note:: You probably do not want to use this! Elements that require access to the list of all other elements should generally use the environment variables exported by disk-image-create below. :param elements: user specified elements :param paths: Alternative ELEMENTS_PATH; default is to use from env :return: A de-duplicated list of tuples [(element, path), (element, path) ...] with all elements and their dependents, including any transitive dependencies. """ elements = _get_elements(elements, paths) return [(element.name, element.path) for element in elements] def expand_dependencies(user_elements, element_dirs): """Deprecated method for expanding element dependencies. .. warning:: DO NOT USE THIS FUNCTION. For compatibility reasons, this function does not provide paths to the returned elements. This means the caller must process override rules if two elements with the same name appear in element_dirs :param user_elements: iterable enumerating the elements a user requested :param element_dirs: The ELEMENTS_PATH to process :return: a set containing user_elements and all dependent elements including any transitive dependencies. """ logger.warning("expand_dependencies() deprecated, use get_elements") elements = _get_elements(user_elements, element_dirs) return set([element.name for element in elements]) def _output_env_vars(elements): """Output eval-able bash strings for IMAGE_ELEMENT vars :param elements: list of Element objects to represent """ # first the "legacy" environment variable that just lists the # elements print("export IMAGE_ELEMENT='%s'" % ' '.join([element.name for element in elements])) # Then YAML output = {} for element in elements: output[element.name] = element.path print("export IMAGE_ELEMENT_YAML='%s'" % yaml.safe_dump(output)) # Then bash array. Unfortunately, bash can't export array # variables. So we take a compromise and produce an exported # function that outputs the string to re-create the array. # You can then simply do # eval declare -A element_array=$(get_image_element_array) # and you have it. output = "" for element in elements: output += '[%s]=%s ' % (element.name, element.path) print("function get_image_element_array {\n" " echo \"%s\"\n" "};\n" "export -f get_image_element_array;" % output) def main(): diskimage_builder.logging_config.setup() parser = argparse.ArgumentParser() parser.add_argument('elements', nargs='+', help='display dependencies of the given elements') parser.add_argument('--env', '-e', action='store_true', default=False, help=('Output eval-able bash strings for ' 'IMAGE_ELEMENT variables')) args = parser.parse_args(sys.argv[1:]) elements = _get_elements(args.elements) if args.env: _output_env_vars(elements) else: # deprecated compatibility output; doesn't include paths. print(' '.join([element.name for element in elements])) return 0 if __name__ == "__main__": main() diskimage-builder-2.11.0/diskimage_builder/elements/000077500000000000000000000000001324273057700224455ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/apt-conf/000077500000000000000000000000001324273057700241545ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/apt-conf/README.rst000066400000000000000000000006401324273057700256430ustar00rootroot00000000000000======== apt-conf ======== This element overrides the default apt.conf for APT based systems. Environment Variables --------------------- DIB_APT_CONF: :Required: No :Default: None :Description: To override `DIB_APT_CONF`, set it to the path to your apt.conf. The new apt.conf will take effect at build time and run time. :Example: ``DIB_APT_CONF=/etc/apt/apt.conf`` diskimage-builder-2.11.0/diskimage_builder/elements/apt-conf/extra-data.d/000077500000000000000000000000001324273057700264305ustar00rootroot0000000000000099-override-default-apt-conf000077500000000000000000000011201324273057700334760ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/apt-conf/extra-data.d#!/bin/bash # Override the default /etc/apt/apt.conf with $DIB_APT_CONF if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # exit directly if DIB_APT_CONF is not defined properly if [ -z "${DIB_APT_CONF:-}" ] ; then echo "DIB_APT_CONF is not set - no apt.conf will be copied in" exit 0 elif [ ! -f "$DIB_APT_CONF" ] ; then echo "$DIB_APT_CONF is not a valid apt.conf file." echo "You should assign a proper apt.conf file in DIB_APT_CONF" exit 1 fi # copy the apt.conf to cloudimg sudo cp -L -f $DIB_APT_CONF $TMP_MOUNT_PATH/etc/apt/apt.conf diskimage-builder-2.11.0/diskimage_builder/elements/apt-preferences/000077500000000000000000000000001324273057700255305ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/apt-preferences/README.rst000066400000000000000000000012631324273057700272210ustar00rootroot00000000000000=============== apt-preferences =============== This element generates the APT preferences file based on the provided manifest provided by the :doc:`../manifests/README` element. The APT preferences file can be used to control which versions of packages will be selected for installation. APT uses a priority system to make this determination. For more information about APT preferences, see the apt_preferences(5) man page. Environment Variables --------------------- DIB_DPKG_MANIFEST: :Required: No :Default: None :Description: The manifest file to generate the APT preferences file from. :Example: ``DIB\_DPKG\_MANIFEST=~/image.d/dib-manifests/dib-manifest-dpkg-image`` diskimage-builder-2.11.0/diskimage_builder/elements/apt-preferences/extra-data.d/000077500000000000000000000000001324273057700300045ustar00rootroot0000000000000099-set-apt-package-pins000077500000000000000000000031431324273057700340300ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/apt-preferences/extra-data.d#!/bin/bash # # Copyright 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # exit directly if DIB_DPKG_MANIFEST is not defined properly if [ -z "${DIB_DPKG_MANIFEST:-}" ]; then echo "DIB_DPKG_MANIFEST must be set to the location of a manifest file you wish to use" exit 0 elif [ ! -f "$DIB_DPKG_MANIFEST" -o ! -s "$DIB_DPKG_MANIFEST" ]; then echo "$DIB_DPKG_MANIFEST is not a valid manifest file." echo "You should assign a proper manifest file in DIB_DPKG_MANIFEST" exit 1 fi DIB_DPKG_MANIFEST=$(readlink -f $DIB_DPKG_MANIFEST) # Create the preferences file from the given manifest outfile=$(mktemp) for package in $(jq -r ".packages[].package" $DIB_DPKG_MANIFEST); do version=$(jq -r ".packages[] | select(.package == \"${package}\") |\ .version" $DIB_DPKG_MANIFEST) cat << EOF >> $outfile Package: ${package} Pin: version ${version} Pin-Priority: 1001 EOF done if [ -s $outfile ]; then sudo mv $outfile $TMP_MOUNT_PATH/etc/apt/preferences else rm $outfile fi diskimage-builder-2.11.0/diskimage_builder/elements/apt-sources/000077500000000000000000000000001324273057700247125ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/apt-sources/README.rst000066400000000000000000000007611324273057700264050ustar00rootroot00000000000000=========== apt-sources =========== Specify an apt sources.list file which is used during image building and then remains on the image when it is run. Environment Variables --------------------- DIB_APT_SOURCES :Required: No :Default: None (Does not replace sources.list file) :Description: Path to a file on the build host which is used in place of ``/etc/apt/sources.list`` :Example: ``DIB_APT_SOURCES=/etc/apt/sources.list`` will use the same sources.list as the build host. diskimage-builder-2.11.0/diskimage_builder/elements/apt-sources/extra-data.d/000077500000000000000000000000001324273057700271665ustar00rootroot0000000000000099-override-default-apt-sources000077500000000000000000000013751324273057700350060ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/apt-sources/extra-data.d#!/bin/bash # Override the default /etc/apt/sources.list with $DIB_APT_SOURCES if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # exit directly if DIB_APT_SOURCES is not defined properly if [ -z "${DIB_APT_SOURCES:-}" ] ; then echo "DIB_APT_SOURCES must be set to the location of a sources.list file you wish to use" exit 0 elif [ ! -f "$DIB_APT_SOURCES" -o ! -s "$DIB_APT_SOURCES" ] ; then echo "$DIB_APT_SOURCES is not a valid sources.list file." echo "You should assign proper sources.list file in DIB_APT_SOURCES" exit 1 fi DIB_APT_SOURCES=`readlink -f $DIB_APT_SOURCES` # copy the sources.list to cloudimg pushd $TMP_MOUNT_PATH/etc/apt/ sudo cp -f $DIB_APT_SOURCES sources.list # dib-lint: safe_sudo popd diskimage-builder-2.11.0/diskimage_builder/elements/apt-sources/test-elements/000077500000000000000000000000001324273057700275035ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/apt-sources/test-elements/test-sources/000077500000000000000000000000001324273057700321435ustar00rootroot00000000000000element-deps000066400000000000000000000000511324273057700343650ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/apt-sources/test-elements/test-sourcesbase openstack-ci-mirrors ubuntu-minimal environment.d/000077500000000000000000000000001324273057700346525ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/apt-sources/test-elements/test-sources00-set-apt-sources000066400000000000000000000000611324273057700400450ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/apt-sources/test-elements/test-sources/environment.dDIB_APT_SOURCES=$(mktemp) export DIB_APT_SOURCES extra-data.d/000077500000000000000000000000001324273057700343405ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/apt-sources/test-elements/test-sources00-write-apt-sources000077500000000000000000000002001324273057700400700ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/apt-sources/test-elements/test-sources/extra-data.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail echo "testdata" > $DIB_APT_SOURCES pre-install.d/000077500000000000000000000000001324273057700345405ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/apt-sources/test-elements/test-sources00-test-apt-sources000077500000000000000000000004001324273057700401170ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/apt-sources/test-elements/test-sources/pre-install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eux set -o pipefail echo "Verifying apt sources.list content" [ -f /etc/apt/sources.list ] [ "$(cat /etc/apt/sources.list)" = "testdata" ] touch /tmp/dib-test-should-fail && exit 1 diskimage-builder-2.11.0/diskimage_builder/elements/baremetal/000077500000000000000000000000001324273057700244015ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/baremetal/README.rst000066400000000000000000000010001324273057700260570ustar00rootroot00000000000000.. _element-baremetal: ========= baremetal ========= This is the baremetal (IE: real hardware) element. Does the following: * extracts the kernel and initial ramdisk of the built image. Optional parameters: * DIB_BAREMETAL_KERNEL_PATTERN and DIB_BAREMETAL_INITRD_PATTERN may be supplied to specify which kernel files are preferred; this can be of use when using custom kernels that don't fit the standard naming patterns. Both variables must be provided in order for them to have any effect. diskimage-builder-2.11.0/diskimage_builder/elements/baremetal/cleanup.d/000077500000000000000000000000001324273057700262525ustar00rootroot0000000000000099-extract-kernel-and-ramdisk000077500000000000000000000022601324273057700335000ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/baremetal/cleanup.d#!/bin/bash # # Copyright 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # dib-lint: disable=safe_sudo if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail [ -n "$TARGET_ROOT" ] source $_LIB/img-functions # Dig up the initrd and kernel to use. select_boot_kernel_initrd $TARGET_ROOT sudo cp $BOOTDIR/$KERNEL ${IMAGE_NAME}.vmlinuz sudo cp $BOOTDIR/$RAMDISK ${IMAGE_NAME}.initrd sudo chmod a+r ${IMAGE_NAME}.vmlinuz sudo chmod a+r ${IMAGE_NAME}.initrd if [ -f $TARGET_ROOT/dib-signed-kernel-version ] ; then echo "Removing $TARGET_ROOT/dib-signed-kernel-version" sudo rm -f $TARGET_ROOT/dib-signed-kernel-version fi diskimage-builder-2.11.0/diskimage_builder/elements/base/000077500000000000000000000000001324273057700233575ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/base/README.rst000066400000000000000000000017361324273057700250550ustar00rootroot00000000000000==== base ==== This is the base element. Almost all users will want to include this in their disk image build, as it includes a lot of useful functionality. The `DIB_CLOUD_INIT_ETC_HOSTS` environment variable can be used to customize cloud-init's management of `/etc/hosts`: * If the variable is set to something, write that value as cloud-init's manage_etc_hosts. * If the variable is set to an empty string, don't create manage_etc_hosts setting (cloud-init will use its default value). * If the variable is not set, use "localhost" for now. Later, not setting the variable will mean using cloud-init's default. (To preserve diskimage-builder's current default behavior in the future, set the variable to "localhost" explicitly.) Notes: * If you are getting warnings during the build about your locale being missing, consider installing/generating the relevant locale. This may be as simple as having language-pack-XX installed in the pre-install stage diskimage-builder-2.11.0/diskimage_builder/elements/base/element-deps000066400000000000000000000000461324273057700256640ustar00rootroot00000000000000dib-init-system install-types pkg-map diskimage-builder-2.11.0/diskimage_builder/elements/base/install.d/000077500000000000000000000000001324273057700252475ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/base/install.d/00-baseline-environment000077500000000000000000000003121324273057700315320ustar00rootroot00000000000000#!/bin/bash # These are useful, or at worst not harmful, for all images we build. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail install-packages -m base iscsi_package diskimage-builder-2.11.0/diskimage_builder/elements/base/install.d/00-up-to-date000077500000000000000000000003111324273057700273640ustar00rootroot00000000000000#!/bin/bash # Fully upgrade everything on the system (if the package manager knows how to # do it). if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail install-packages -u diskimage-builder-2.11.0/diskimage_builder/elements/base/install.d/10-cloud-init000077500000000000000000000005601324273057700274630ustar00rootroot00000000000000#!/bin/bash # Tweak the stock ubuntu cloud-init config if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # cloud-init May not actually be installed mkdir -p /etc/cloud/cloud.cfg.d if [ -n "${DIB_CLOUD_INIT_ETC_HOSTS:-}" ]; then dd of=/etc/cloud/cloud.cfg.d/10_etc_hosts.cfg << EOF manage_etc_hosts: $DIB_CLOUD_INIT_ETC_HOSTS EOF fi diskimage-builder-2.11.0/diskimage_builder/elements/base/pkg-map000066400000000000000000000015131324273057700246360ustar00rootroot00000000000000{ "family": { "redhat": { "iscsi_package": "iscsi-initiator-utils" }, "gentoo": { "curl": "net-misc/curl", "dhcp_client": "net-misc/dhcp", "extlinux": "sys-boot/syslinux", "git": "dev-vcs/git", "grub_bios": "sys-boot/grub", "grub-pc": "sys-boot/grub", "ironic-python-agent": "", "iscsi_package": "sys-block/open-iscsi", "isc-dhcp-client": "net-misc/dhcp", "isolinux": "", "ncat": "net-analyzer/netcat", "qemu-utils": "app-emulation/qemu", "python-dev": "", "PyYAML": "dev-python/pyyaml", "syslinux": "sys-boot/syslinux", "syslinux-common": "", "tftp": "net-ftp/tftp-hpa", "tgt": "sys-block/tgt" }, "suse": { "qemu-utils": "qemu-tools" } }, "default": { "iscsi_package": "open-iscsi" } } diskimage-builder-2.11.0/diskimage_builder/elements/base/pre-install.d/000077500000000000000000000000001324273057700260335ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/base/pre-install.d/03-baseline-tools000077500000000000000000000006611324273057700311240ustar00rootroot00000000000000#!/bin/bash # Install baseline packages and tools. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail case $DISTRO_NAME in 'ubuntu'|'debian') # Note: add-apt-repository would be nice for RPM platforms too - so when we # need something like it, create a wrapper in dpkg/bin and fedora/bin. apt-get -y update install-packages software-properties-common ;; esac diskimage-builder-2.11.0/diskimage_builder/elements/bootloader/000077500000000000000000000000001324273057700245775ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/bootloader/README.rst000066400000000000000000000011261324273057700262660ustar00rootroot00000000000000========== bootloader ========== Installs ``grub[2]`` on boot partition on the system. In case GRUB2 is not available in the system, a fallback to Extlinux will happen. It's also possible to enforce the use of Extlinux by exporting a ``DIB_EXTLINUX`` variable to the environment. Arguments ========= * ``DIB_GRUB_TIMEOUT`` sets the ``grub`` menu timeout. It defaults to 5 seconds. Set this to 0 (no timeout) for fast boot times. * ``DIB_BOOTLOADER_DEFAULT_CMDLINE`` sets the CMDLINE parameters that are appended to the grub.cfg configuration. It defaults to 'nofb nomodeset vga=normal' diskimage-builder-2.11.0/diskimage_builder/elements/bootloader/cleanup.d/000077500000000000000000000000001324273057700264505ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/bootloader/cleanup.d/51-bootloader000077500000000000000000000027001324273057700307520ustar00rootroot00000000000000#!/bin/bash # # Copyright 2014 Hewlett-Packard Development Company, L.P. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # dib-lint: disable=safe_sudo if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail [ -n "$TARGET_ROOT" ] source $_LIB/img-functions if [ -d $TARGET_ROOT/boot/extlinux ] ; then CONF=$TARGET_ROOT/boot/extlinux/extlinux.conf elif [ -d $TARGET_ROOT/boot/syslinux ] ; then CONF=$TARGET_ROOT/boot/syslinux/syslinux.cfg else exit 0 fi # Dig up the initrd and kernel to use. select_boot_kernel_initrd $TARGET_ROOT # Serial console on Power is hvc0 if [[ "powerpc ppc64 ppc64le" =~ "$ARCH" ]] ; then SERIAL_CONSOLE="hvc0" else SERIAL_CONSOLE="ttyS0,115200" fi sudo sh -c "cat > $CONF <<_EOF_ DEFAULT linux LABEL linux KERNEL /boot/$KERNEL APPEND ro root=LABEL=${DIB_ROOT_LABEL} console=tty0 console=${SERIAL_CONSOLE} nofb nomodeset vga=normal INITRD /boot/$RAMDISK _EOF_" diskimage-builder-2.11.0/diskimage_builder/elements/bootloader/environment.d/000077500000000000000000000000001324273057700273655ustar00rootroot0000000000000010-bootloader-default-cmdline000066400000000000000000000001451324273057700346340ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/bootloader/environment.dexport DIB_BOOTLOADER_DEFAULT_CMDLINE=${DIB_BOOTLOADER_DEFAULT_CMDLINE:-"nofb nomodeset vga=normal"} diskimage-builder-2.11.0/diskimage_builder/elements/bootloader/finalise.d/000077500000000000000000000000001324273057700266135ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/bootloader/finalise.d/50-bootloader000077500000000000000000000165271324273057700311300ustar00rootroot00000000000000#!/bin/bash # Configure grub. Note that the various conditionals here are to handle # different distributions gracefully. if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail BOOT_DEV=$IMAGE_BLOCK_DEVICE # All available devices, handy for some bootloaders... declare -A DEVICES eval DEVICES=( $IMAGE_BLOCK_DEVICES ) function install_extlinux { install-packages -m bootloader extlinux echo "Installing Extlinux..." # Find and install mbr.bin for MBR in /usr/share/syslinux/mbr.bin /usr/lib/syslinux/mbr.bin \ /usr/lib/extlinux/mbr.bin /usr/lib/EXTLINUX/mbr.bin ; do if [ -f $MBR ]; then break fi done if [ ! -f $MBR ]; then echo "mbr.bin (from EXT/SYSLINUX) not found." exit 1 fi dd if=$MBR of=$BOOT_DEV # Find any pre-created extlinux install directory for EXTDIR in /boot/extlinux /boot/syslinux ; do if [ -d $EXTDIR ] ; then break fi done if [ ! -d $EXTDIR ] ; then # No install directory found so default to /boot/syslinux EXTDIR=/boot/syslinux mkdir -p $EXTDIR fi # Finally install extlinux extlinux --install $EXTDIR } function install_grub2 { # Check for offline installation of grub if [ -f "/tmp/grub/install" ] ; then source /tmp/grub/install # Right now we can't use pkg-map to branch by arch, so tag an architecture # specific virtual package so we can install the rigth thing based on # distribution. elif [[ "$ARCH" =~ "ppc" ]]; then install-packages -m bootloader grub-ppc64 else install-packages -m bootloader grub-pc fi # XXX: grub-probe on the nbd0/loop0 device returns nothing - workaround, manually # specify modules. https://bugs.launchpad.net/ubuntu/+source/grub2/+bug/1073731 GRUBNAME=$(type -p grub-install) || echo "trying grub2-install" if [ -z "$GRUBNAME" ]; then GRUBNAME=$(type -p grub2-install) fi # If no GRUB2 is found, fallback to extlinux if [ -z "$GRUBNAME" ] || [ $($GRUBNAME --version | grep "0.97" | wc -l) -ne 0 ]; then echo "No GRUB2 found. Fallback to Extlinux..." install_extlinux exit 0 fi echo "Installing GRUB2..." # We need --force so grub does not fail due to being installed on the # root partition of a block device. GRUB_OPTS=${GRUB_OPTS:-"--force"} # XXX: This is buggy: # - --target=i386-pc is invalid for non-i386/amd64 architectures # - and for UEFI too. # GRUB_OPTS="$GRUB_OPTS --target=i386-pc" if [[ ! $GRUB_OPTS == *--target* ]] && [[ $($GRUBNAME --version) =~ ' 2.' ]]; then # /sys/ comes from the host machine. If the host machine is using EFI # but the image being built doesn't have EFI boot-images installed we # should set the --target to use a BIOS-based boot-image. # # * --target tells grub what's the target platform # * the boot images are placed in /usr/lib/grub/- # * i386-pc is used for BIOS-based machines # http://www.gnu.org/software/grub/manual/grub.html#Installation # if [ -d /sys/firmware/efi ]; then if [ ! -d /usr/lib/grub/*-efi ]; then case $ARCH in "x86_64"|"amd64") GRUB_OPTS="$GRUB_OPTS --target=i386-pc" ;; "i386") target=i386-pc if [ -e /proc/device-tree ]; then for x in /proc/device-tree/*; do if [ -e "$x" ]; then target="i386-ieee1275" fi done fi GRUB_OPTS="$GRUB_OPTS --target=$target" ;; esac fi fi fi if [[ "$ARCH" =~ "ppc" ]] ; then # For PPC (64-Bit regardless of Endian-ness), we use the "boot" # partition as the one to point grub-install to, not the loopback # device. ppc has a dedicated PReP boot partition. # For grub2 < 2.02~beta3 this needs to be a /dev/mapper/... node after # that a dev/loopXpN node will work fine. $GRUBNAME --modules="part_msdos" $GRUB_OPTS ${DEVICES[boot]} --no-nvram else $GRUBNAME --modules="biosdisk part_msdos" $GRUB_OPTS $BOOT_DEV fi # This might be better factored out into a per-distro 'install-bootblock' # helper. if [ -d /boot/grub2 ]; then GRUB_CFG=/boot/grub2/grub.cfg elif [ -d /boot/grub ]; then GRUB_CFG=/boot/grub/grub.cfg fi # Override the root device to the default label, and disable uuid # lookup. echo "GRUB_DEVICE=LABEL=${DIB_ROOT_LABEL}" >> /etc/default/grub echo 'GRUB_DISABLE_LINUX_UUID=true' >> /etc/default/grub echo "GRUB_TIMEOUT=${DIB_GRUB_TIMEOUT:-5}" >>/etc/default/grub echo 'GRUB_TERMINAL="serial console"' >>/etc/default/grub echo 'GRUB_GFXPAYLOAD_LINUX=text' >>/etc/default/grub # Serial console on Power is hvc0 if [[ "powerpc ppc64 ppc64le" =~ "$ARCH" ]]; then SERIAL_CONSOLE="hvc0" else SERIAL_CONSOLE="ttyS0,115200" fi GRUB_CMDLINE_LINUX_DEFAULT="console=tty0 console=${SERIAL_CONSOLE} no_timer_check" echo "GRUB_CMDLINE_LINUX_DEFAULT=\"${GRUB_CMDLINE_LINUX_DEFAULT}\"" >>/etc/default/grub echo 'GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"' >>/etc/default/grub if type grub2-mkconfig >/dev/null; then GRUB_MKCONFIG="grub2-mkconfig -o $GRUB_CFG" else GRUB_MKCONFIG="grub-mkconfig -o $GRUB_CFG" fi DISTRO_NAME=${DISTRO_NAME:-} case $DISTRO_NAME in 'ubuntu'|'debian') sed -i -e "s/\(^GRUB_CMDLINE_LINUX.*\)\"$/\1 ${DIB_BOOTLOADER_DEFAULT_CMDLINE}\"/" /etc/default/grub GRUB_MKCONFIG=update-grub ;; 'fedora'|'centos7'|'centos') echo "GRUB_CMDLINE_LINUX=\"${DIB_BOOTLOADER_DEFAULT_CMDLINE}\"" >>/etc/default/grub ;; 'opensuse') sed -i -e "s/\(^GRUB_CMDLINE_LINUX.*\)\"$/\1 ${DIB_BOOTLOADER_DEFAULT_CMDLINE}\"/" /etc/default/grub ;; esac # os-prober leaks /dev/sda into config file in dual-boot host # Disable grub-os-prober to avoid the issue while running # grub-mkconfig # Setting a flag to track whether the entry is already there in grub config PROBER_DISABLED= if ! grep -qe "^\s*GRUB_DISABLE_OS_PROBER=true" /etc/default/grub; then PROBER_DISABLED=true echo 'GRUB_DISABLE_OS_PROBER=true' >> /etc/default/grub fi $GRUB_MKCONFIG # Remove the fix to disable os_prober if [ -n "$PROBER_DISABLED" ]; then sed -i '$d' /etc/default/grub fi # grub-mkconfig generates a config with the device in it, # This shouldn't be needed, but old code has bugs DIB_RELEASE=${DIB_RELEASE:-} if [ "$DIB_RELEASE" = 'wheezy' ]; then sed -i "s%search --no.*%%" $GRUB_CFG sed -i "s%set root=.*%set root=(hd0,1)%" $GRUB_CFG fi # Fix efi specific instructions in grub config file if [ -d /sys/firmware/efi ]; then sed -i 's%\(initrd\|linux\)efi /boot%\1 /boot%g' $GRUB_CFG fi } DIB_EXTLINUX=${DIB_EXTLINUX:-0} if [ "$DIB_EXTLINUX" != "0" ]; then install_extlinux else install_grub2 fi diskimage-builder-2.11.0/diskimage_builder/elements/bootloader/pkg-map000066400000000000000000000007661324273057700260670ustar00rootroot00000000000000{ "family": { "gentoo": { "dkms_package": "", "extlinux": "syslinux", "grub-pc": "grub" }, "suse": { "dkms_package": "", "extlinux": "syslinux", "grub-pc": "grub2" }, "redhat": { "extlinux": "syslinux-extlinux", "grub-pc": "grub2-tools grub2", "grub-ppc64": "grub2-tools grub2" } }, "default": { "dkms_package": "dkms", "extlinux": "extlinux", "grub-pc": "grub-pc", "grub-ppc64": "grub-ieee1275" } } diskimage-builder-2.11.0/diskimage_builder/elements/cache-url/000077500000000000000000000000001324273057700243105ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/cache-url/README.rst000066400000000000000000000001251324273057700257750ustar00rootroot00000000000000========= cache-url ========= A helper script to download images into a local cache. diskimage-builder-2.11.0/diskimage_builder/elements/cache-url/__init__.py000066400000000000000000000000001324273057700264070ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/cache-url/bin/000077500000000000000000000000001324273057700250605ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/cache-url/bin/cache-url000077500000000000000000000071431324273057700266560ustar00rootroot00000000000000#!/bin/bash # Copyright 2012 Hewlett-Packard Development Company, L.P. # Copyright 2013 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # Download a URL to a local cache # e.g. cache-url http://.../foo ~/.cache/image-create/foo SCRIPT_NAME=$(basename $0) SCRIPT_HOME=$(dirname $0) FORCE_REVALIDATE=0 function show_options () { echo "Usage: $SCRIPT_NAME [options] " echo echo "Download a URL and cache it to a specified location." echo "Subsequent requests will compare the last modified date" echo "of the upstream file to determine whether it needs to be" echo "downloaded again." echo echo "Options:" echo " -f -- force upstream caches to fetch a new copy of the file" echo " -h -- show this help" echo exit $1 } TEMP=$(getopt -o hf -n $SCRIPT_NAME -- "$@") if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi # Note the quotes around `$TEMP': they are essential! eval set -- "$TEMP" while true ; do case "$1" in -h|"-?") show_options 0;; -f) FORCE_REVALIDATE=1; shift 1;; --) shift; break;; *) echo "Error: unsupported option $1." ; exit 1 ;; esac done url=$1 dest=$2 time_cond= curl_opts="" if [ -z $url -o -z $dest ] ; then show_options 1 fi if [ -p $dest ]; then type="fifo" tmp=$(mktemp --tmpdir download.XXXXXXXX) else type="normal" mkdir -p $(dirname $dest) tmp=$(mktemp $(dirname $dest)/.download.XXXXXXXX) fi if [ "$FORCE_REVALIDATE" = "1" ]; then curl_opts="-H 'Pragma: no-cache, must-revalidate' -H 'Cache-Control: no-cache, must-revalidate'" success="Downloaded and cached $url, having forced upstream caches to revalidate" elif [ -f $dest -a -s $dest ] ; then time_cond="-z $dest" success="Server copy has changed. Using server version of $url" else success="Downloaded and cached $url for the first time" fi rcode=$(curl -v -L -o $tmp -w '%{http_code}' --connect-timeout 10 $curl_opts $url $time_cond) if [ "$rcode" == "200" -o "${url:0:7}" == "file://" ] ; then # In cases where servers ignore the Modified time, # curl cancels the download, outputs a 200 and leaves # the output file untouched, we don't want this empty file. if [ -n "$time_cond" -a ! -s $tmp ] ; then echo "Ignoring empty file returned by curl. Using locally cached $url" rm -f $tmp else echo $success if [ "fifo" = "$type" ]; then cp $tmp $dest rm $tmp else mv $tmp $dest fi fi # 213 is the response to a ftp MDTM command, curl outputs a 213 as the status # if the url redirected to a ftp server and Not-Modified elif [ "$rcode" = "304" -o "$rcode" = "213" ] ; then echo "Server copy has not changed. Using locally cached $url" rm -f $tmp else echo "Server returned an unexpected response code. [$rcode]" rm -f $tmp # expose some error codes so the calling process might know what happened if [ "$rcode" = "404" ] ; then exit 44 fi exit 1 fi diskimage-builder-2.11.0/diskimage_builder/elements/cache-url/tests/000077500000000000000000000000001324273057700254525ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/cache-url/tests/__init__.py000066400000000000000000000000001324273057700275510ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/cache-url/tests/test_cache_url.py000066400000000000000000000030461324273057700310130ustar00rootroot00000000000000# Copyright 2014 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import tempfile import time from diskimage_builder.tests import base class TestCacheUrl(base.ScriptTestBase): def test_cache_url_caches(self): tempdir = tempfile.mkdtemp() target = os.path.join(tempdir, 'target') source = 'http://fake/url' # Write fake data to the target file and return success self._stub_script('curl', 'echo "test" > ${3:7:100}\necho 200') self._run_command(['elements/cache-url/bin/cache-url', source, target]) self.assertTrue(os.path.exists(target)) modification_time = os.path.getmtime(target) # Make sure that the timestamp would change if the file does time.sleep(1) self._stub_script('curl', 'echo "304"') self._run_command(['elements/cache-url/bin/cache-url', source, target]) self.assertEqual(modification_time, os.path.getmtime(target)) diskimage-builder-2.11.0/diskimage_builder/elements/centos-minimal/000077500000000000000000000000001324273057700253645ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/centos-minimal/README.rst000066400000000000000000000011341324273057700270520ustar00rootroot00000000000000============== centos-minimal ============== Create a minimal image based on CentOS 7. Use of this element will require 'yum' and 'yum-utils' to be installed on Ubuntu and Debian. Nothing additional is needed on Fedora or CentOS. By default, ``DIB_YUM_MINIMAL_CREATE_INTERFACES`` is set to enable the creation of ``/etc/sysconfig/network-scripts/ifcfg-eth[0|1]`` scripts to enable DHCP on the ``eth0`` & ``eth1`` interfaces. If you do not have these interfaces, or if you are using something else to setup the network such as cloud-init, glean or network-manager, you would want to set this to ``0``. diskimage-builder-2.11.0/diskimage_builder/elements/centos-minimal/element-deps000066400000000000000000000000151324273057700276650ustar00rootroot00000000000000yum-minimal diskimage-builder-2.11.0/diskimage_builder/elements/centos-minimal/element-provides000066400000000000000000000000211324273057700305620ustar00rootroot00000000000000operating-system diskimage-builder-2.11.0/diskimage_builder/elements/centos-minimal/environment.d/000077500000000000000000000000001324273057700301525ustar00rootroot0000000000000010-centos-distro-name.bash000066400000000000000000000005231324273057700346630ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/centos-minimal/environment.dexport DISTRO_NAME=centos export DIB_RELEASE=${DIB_RELEASE:-7} # by default, enable DHCP configuration of eth0 & eth1 in network # scripts. See yum-minimal for full details export DIB_YUM_MINIMAL_CREATE_INTERFACES=${DIB_YUM_MINIMAL_CREATE_INTERFACES:-1} # Useful for elements that work with fedora (dnf) & centos export YUM=${YUM:-yum} diskimage-builder-2.11.0/diskimage_builder/elements/centos-minimal/test-elements/000077500000000000000000000000001324273057700301555ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/centos-minimal/test-elements/build-succeeds/000077500000000000000000000000001324273057700330505ustar00rootroot00000000000000README.rst000066400000000000000000000000541324273057700344570ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/centos-minimal/test-elements/build-succeedsVerify we can build a centos-minimal image. element-deps000066400000000000000000000000241324273057700352720ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/centos-minimal/test-elements/build-succeedsopenstack-ci-mirrorsdiskimage-builder-2.11.0/diskimage_builder/elements/centos-minimal/yum.repos.d/000077500000000000000000000000001324273057700275475ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/centos-minimal/yum.repos.d/yum.repo000066400000000000000000000003771324273057700312570ustar00rootroot00000000000000[centos] name=CentOS-$releasever - Base mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os gpgcheck=0 #baseurl=http://mirror.centos.org/centos/$releasever/os/$basearch/ #gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 diskimage-builder-2.11.0/diskimage_builder/elements/centos7/000077500000000000000000000000001324273057700240275ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/centos7/README.rst000066400000000000000000000014201324273057700255130ustar00rootroot00000000000000======= centos7 ======= Use Centos 7 cloud images as the baseline for built disk images. For further details see the redhat-common README. DIB_DISTRIBUTION_MIRROR: :Required: No :Default: None :Description: To use a CentOS Yum mirror, set this variable to the mirror URL before running bin/disk-image-create. This URL should point to the directory containing the ``5/6/7`` directories. :Example: ``DIB_DISTRIBUTION_MIRROR=http://amirror.com/centos`` DIB_CLOUD_IMAGES :Required: No :Description: Set the desired URL to fetch the images from. ppc64le: Currently the CentOS community is working on providing the ppc64le images until then you'll need to set this to a local image file. diskimage-builder-2.11.0/diskimage_builder/elements/centos7/element-deps000066400000000000000000000000731324273057700263340ustar00rootroot00000000000000cache-url redhat-common rpm-distro source-repositories yum diskimage-builder-2.11.0/diskimage_builder/elements/centos7/element-provides000066400000000000000000000000211324273057700272250ustar00rootroot00000000000000operating-system diskimage-builder-2.11.0/diskimage_builder/elements/centos7/environment.d/000077500000000000000000000000001324273057700266155ustar00rootroot0000000000000010-centos7-distro-name.bash000066400000000000000000000002171324273057700334150ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/centos7/environment.dexport DISTRO_NAME=centos7 export DIB_RELEASE=GenericCloud # Useful for elements that work with fedora (dnf) & centos export YUM=${YUM:-yum} diskimage-builder-2.11.0/diskimage_builder/elements/centos7/pre-install.d/000077500000000000000000000000001324273057700265035ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/centos7/pre-install.d/01-set-centos-mirror000077500000000000000000000007131324273057700322440ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-} [ -n "$DIB_DISTRIBUTION_MIRROR" ] || exit 0 # Only set the mirror for the Base, Extras and Updates repositories # The others arn't enabled and do not exist on all mirrors sed -e "s|^#baseurl=http://mirror.centos.org/centos|baseurl=$DIB_DISTRIBUTION_MIRROR|;/^mirrorlist=/d" -i /etc/yum.repos.d/CentOS-Base.repo diskimage-builder-2.11.0/diskimage_builder/elements/centos7/root.d/000077500000000000000000000000001324273057700252345ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/centos7/root.d/10-centos7-cloud-image000077500000000000000000000026361324273057700311550ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail [ -n "$ARCH" ] [ -n "$TARGET_ROOT" ] if [[ "amd64 x86_64" =~ "$ARCH" ]]; then ARCH="x86_64" DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-http://cloud.centos.org/centos/7/images} elif [[ "arm64 aarch64" =~ "$ARCH" ]]; then ARCH="aarch64" DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-http://cloud.centos.org/altarch/7/images/aarch64} elif [[ "ppc64le" =~ "$ARCH" ]]; then DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-http://cloud.centos.org/altarch/7/images/ppc64le} else echo 'centos7 root element only support the x86_64, aarch64 and ppc64le values for $ARCH' exit 1 fi DIB_LOCAL_IMAGE=${DIB_LOCAL_IMAGE:-} if [ -n "$DIB_LOCAL_IMAGE" ]; then IMAGE_LOCATION=$DIB_LOCAL_IMAGE # No need to copy a local image into the cache directory, so just specify # the cached path as the original path. CACHED_IMAGE=$IMAGE_LOCATION BASE_IMAGE_FILE=$(basename $DIB_LOCAL_IMAGE) BASE_IMAGE_TAR=$BASE_IMAGE_FILE.tgz else DIB_RELEASE=${DIB_RELEASE:-GenericCloud} DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES} BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-CentOS-7-${ARCH}-$DIB_RELEASE.qcow2.xz} BASE_IMAGE_TAR=$BASE_IMAGE_FILE.tgz IMAGE_LOCATION=$DIB_CLOUD_IMAGES/$BASE_IMAGE_FILE CACHED_IMAGE=$DIB_IMAGE_CACHE/$BASE_IMAGE_FILE fi $TMP_HOOKS_PATH/bin/extract-image $BASE_IMAGE_FILE $BASE_IMAGE_TAR $IMAGE_LOCATION $CACHED_IMAGE diskimage-builder-2.11.0/diskimage_builder/elements/centos7/test-elements/000077500000000000000000000000001324273057700266205ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/centos7/test-elements/build-succeeds/000077500000000000000000000000001324273057700315135ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/centos7/test-elements/build-succeeds/README.rst000066400000000000000000000000001324273057700331700ustar00rootroot00000000000000element-deps000066400000000000000000000000321324273057700337340ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/centos7/test-elements/build-succeedsbase openstack-ci-mirrors diskimage-builder-2.11.0/diskimage_builder/elements/cleanup-kernel-initrd/000077500000000000000000000000001324273057700266415ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/cleanup-kernel-initrd/README.rst000066400000000000000000000001561324273057700303320ustar00rootroot00000000000000===================== cleanup-kernel-initrd ===================== Remove unused kernel/initrd from the image. diskimage-builder-2.11.0/diskimage_builder/elements/cleanup-kernel-initrd/cleanup.d/000077500000000000000000000000001324273057700305125ustar00rootroot0000000000000099-cleanup-kernel-initrd000077500000000000000000000030301324273057700350100ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/cleanup-kernel-initrd/cleanup.d#!/bin/bash # # Copyright 2014 Hewlett-Packard Development Company, L.P. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail source $_LIB/img-functions function cleanup_old_kernel_initrd() { KERNEL_VERSION=${KERNEL#vmlinuz-} if [ -n "${BOOTDIR}" ] ; then for file in `ls $BOOTDIR/vmlinuz-* | grep -v $KERNEL_VERSION | grep -v debug` do echo "Removing unnecessary kernels and ramdisk images from image." file=$(basename $file) rm -Rvf $BOOTDIR/*${file#vmlinuz-}* done fi } # Dig up the initrd and kernel. select_boot_kernel_initrd $TARGET_ROOT BOOTDIR=$TARGET_ROOT/boot # Select_boot_kernel has support for redhat/debian/openSUSE # openSUSE has a built-in logic to delete old kernels # Hence cleanup_old_kernel will be used only for redhat and debian derived distros. if [ -f $TARGET_ROOT/etc/redhat-release -o -f $TARGET_ROOT/etc/debian_version ] ; then cleanup_old_kernel_initrd fi diskimage-builder-2.11.0/diskimage_builder/elements/cloud-init-datasources/000077500000000000000000000000001324273057700270275ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/cloud-init-datasources/README.rst000066400000000000000000000016741324273057700305260ustar00rootroot00000000000000====================== cloud-init-datasources ====================== Configures cloud-init to only use an explicit list of data sources. Cloud-init contains a growing collection of data source modules and most are enabled by default. This causes cloud-init to query each data source on first boot. This can cause delays or even boot problems depending on your environment. Including this element without setting `DIB_CLOUD_INIT_DATASOURCES` will cause image builds to fail. Environment Variables --------------------- DIB_CLOUD_INIT_DATASOURCES :Required: Yes :Default: None :Description: A comma-separated list of valid data sources to limit the data sources that will be queried for metadata on first boot. :Example: ``DIB_CLOUD_INIT_DATASOURCES="Ec2"`` will enable only the Ec2 data source. :Example: ``DIB_CLOUD_INIT_DATASOURCES="Ec2, ConfigDrive, OpenStack"`` will enable the Ec2, ConfigDrive and OpenStack data sources. diskimage-builder-2.11.0/diskimage_builder/elements/cloud-init-datasources/install.d/000077500000000000000000000000001324273057700307175ustar00rootroot0000000000000005-set-cloud-init-sources000077500000000000000000000017721324273057700353400ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/cloud-init-datasources/install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail DIB_CLOUD_INIT_DATASOURCES=${DIB_CLOUD_INIT_DATASOURCES:-""} if [ -z "$DIB_CLOUD_INIT_DATASOURCES" ] ; then echo "DIB_CLOUD_INIT_DATASOURCES must be set to a comma-separated list " echo "of cloud-init data sources you wish to use, ie 'Ec2, NoCloud, ConfigDrive'" exit 1 fi if [ -d /etc/cloud/cloud.cfg.d ]; then cat > /etc/cloud/cloud.cfg.d/91-dib-cloud-init-datasources.cfg < /etc/cloud/cloud.cfg.d/92-ec2-datasource.cfg <> /etc/cloud/cloud.cfg' else echo "ssh_pwauth exist. make sure ssh_pwauth enabled" sed -i -e 's/ssh_pwauth: *0/ssh_pwauth: 1/g' /etc/cloud/cloud.cfg sed -i -e 's/ssh_pwauth: *False/ssh_pwauth: True/g' /etc/cloud/cloud.cfg sed -i -e 's/ssh_pwauth: *false/ssh_pwauth: true/g' /etc/cloud/cloud.cfg fi fi fi diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/000077500000000000000000000000001324273057700253135ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/README.rst000066400000000000000000000007541324273057700270100ustar00rootroot00000000000000============== debian-minimal ============== The ``debian-minimal`` element uses debootstrap for generating a minimal image. By default this element creates the latest stable release. The exact setting can be found in the element's ``environment.d`` directory in the variable ``DIB_RELEASE``. If a different release of Debian should be created, the variable ``DIB_RELEASE`` can be set appropriately. Note that this element installs ``systemd-sysv`` as the init system .. element_deps:: diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/element-deps000066400000000000000000000000371324273057700276200ustar00rootroot00000000000000debootstrap dib-python pkg-map diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/element-provides000066400000000000000000000000211324273057700305110ustar00rootroot00000000000000operating-system diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/environment.d/000077500000000000000000000000001324273057700301015ustar00rootroot0000000000000010-debian-minimal.bash000066400000000000000000000020601324273057700337430ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/environment.dexport DISTRO_NAME=debian export DIB_RELEASE=${DIB_RELEASE:-stable} if [ -n "${DIB_DEBIAN_DISTRIBUTION_MIRROR:-}" ]; then DIB_DISTRIBUTION_MIRROR=$DIB_DEBIAN_DISTRIBUTION_MIRROR fi export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://deb.debian.org/debian} export DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main} export DIB_DEBIAN_COMPONENTS_WS=${DIB_DEBIAN_COMPONENTS//,/ } DIB_APT_SOURCES_CONF_DEFAULT=\ "default:deb ${DIB_DISTRIBUTION_MIRROR} ${DIB_RELEASE} ${DIB_DEBIAN_COMPONENTS_WS} backports:deb ${DIB_DISTRIBUTION_MIRROR} ${DIB_RELEASE}-backports ${DIB_DEBIAN_COMPONENTS_WS} updates:deb ${DIB_DISTRIBUTION_MIRROR} ${DIB_RELEASE}-updates ${DIB_DEBIAN_COMPONENTS_WS} security:deb http://security.debian.org/ ${DIB_RELEASE}/updates ${DIB_DEBIAN_COMPONENTS_WS} " if [ "${DIB_RELEASE}" = "testing" -o "${DIB_RELEASE}" = "unstable" ]; then DIB_APT_SOURCES_CONF_DEFAULT="default:deb ${DIB_DISTRIBUTION_MIRROR} ${DIB_RELEASE} ${DIB_DEBIAN_COMPONENTS_WS}" fi export DIB_APT_SOURCES_CONF=${DIB_APT_SOURCES_CONF:-${DIB_APT_SOURCES_CONF_DEFAULT}} diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/package-installs.yaml000066400000000000000000000001511324273057700314160ustar00rootroot00000000000000linux-image-amd64: arch: amd64 linux-image-686: arch: i386 linux-image-arm64: arch: arm64 netbase: diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/root.d/000077500000000000000000000000001324273057700265205ustar00rootroot0000000000000075-debian-minimal-baseinstall000077500000000000000000000045031324273057700337670ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/root.d#!/bin/bash # Copyright (c) 2014 Hewlett-Packard Development Company, L.P. # Copyright (c) 2016 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # # See the License for the specific language governing permissions and # limitations under the License. # dib-lint: disable=safe_sudo if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # Writes the apt sources files. # The description is passed in via line coded elements. # (The approach using associative arrays for configuration faild, # because it looks that there is no way to handle defaults in # this case - and additionally we run with '-u'.) function apt_sources_write { local APT_SOURCES_CONF="$1" sudo mkdir -p $TARGET_ROOT/etc/apt/sources.list.d echo "${APT_SOURCES_CONF}" \ | while read line; do local name=$(echo ${line} | cut -d ":" -f 1) local value=$(echo ${line} | cut -d ":" -f 2-) echo "$value" | sudo tee $TARGET_ROOT/etc/apt/sources.list.d/${name}.list done } sudo mount -t proc none $TARGET_ROOT/proc sudo mount -t sysfs none $TARGET_ROOT/sys trap "sudo umount $TARGET_ROOT/proc; sudo umount $TARGET_ROOT/sys" EXIT apt_get="sudo chroot $TARGET_ROOT /usr/bin/apt-get" apt_sources_write "${DIB_APT_SOURCES_CONF}" # Need to update to retrieve the signed Release file $apt_get update $apt_get clean $apt_get dist-upgrade -y to_install="" # pre-stretch (9.0) brought this in via debootstrap, but init # dependencies have narrowed in the container world, so now we add it # explicitly here so it's brought in early. to_install+="systemd-sysv " # default required to_install+="busybox sudo " if [ "$DIB_PYTHON_VERSION" == "2" ]; then to_install+="python " elif [ "$DIB_PYTHON_VERSION" == "3" ]; then to_install+="python3 " else echo "ERROR: DIB_PYTHON_VERSION is '$DIB_PYTHON_VERSION' but needs to be 2 or 3" exit 1 fi $apt_get install -y $to_install diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/test-elements/000077500000000000000000000000001324273057700301045ustar00rootroot00000000000000stable-build-succeeds/000077500000000000000000000000001324273057700341705ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/test-elementsREADME.rst000066400000000000000000000000631324273057700356560ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/test-elements/stable-build-succeedsVerify we can build a debian-minimal stable image. environment.d/000077500000000000000000000000001324273057700367565ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/test-elements/stable-build-succeeds10-set-distro.bash000066400000000000000000000000641324273057700421300ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/test-elements/stable-build-succeeds/environment.dexport DISTRO_NAME=debian export DIB_RELEASE=stable diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/test-elements/stable-vm/000077500000000000000000000000001324273057700317765ustar00rootroot00000000000000README.rst000066400000000000000000000000661324273057700334100ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/test-elements/stable-vmVerify we can build a debian-minimal stable VM image. element-deps000066400000000000000000000000031324273057700342150ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/test-elements/stable-vmvm environment.d/000077500000000000000000000000001324273057700345055ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/test-elements/stable-vm10-set-distro.bash000066400000000000000000000000641324273057700376570ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/test-elements/stable-vm/environment.dexport DISTRO_NAME=debian export DIB_RELEASE=stable testing-build-succeeds/000077500000000000000000000000001324273057700343735ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/test-elementsREADME.rst000066400000000000000000000000631324273057700360610ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/test-elements/testing-build-succeedsVerify we can build a debian-minimal stable image. environment.d/000077500000000000000000000000001324273057700371615ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/test-elements/testing-build-succeeds10-set-distro.bash000066400000000000000000000000651324273057700423340ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-minimal/test-elements/testing-build-succeeds/environment.dexport DISTRO_NAME=debian export DIB_RELEASE=testing diskimage-builder-2.11.0/diskimage_builder/elements/debian-systemd/000077500000000000000000000000001324273057700253555ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-systemd/README.rst000066400000000000000000000004271324273057700270470ustar00rootroot00000000000000============== debian-systemd ============== You may want to use `systemd` instead of the classic sysv init system. In this case, include this element in your element list. Note that this works with the ``debian`` element, not the ``debian-minimal`` element. .. element_deps:: diskimage-builder-2.11.0/diskimage_builder/elements/debian-systemd/element-deps000066400000000000000000000000071324273057700276570ustar00rootroot00000000000000debian diskimage-builder-2.11.0/diskimage_builder/elements/debian-systemd/root.d/000077500000000000000000000000001324273057700265625ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-systemd/root.d/05-debian-systemd000077500000000000000000000002661324273057700316460ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail cat > ${TARGET_ROOT}/.extra_settings << EOF DIB_DEBIAN_ALT_INIT_PACKAGE=systemd-sysv EOF diskimage-builder-2.11.0/diskimage_builder/elements/debian-upstart/000077500000000000000000000000001324273057700253675ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-upstart/README.rst000066400000000000000000000005461324273057700270630ustar00rootroot00000000000000============== debian-upstart ============== By default Debian will use sysvinit for booting. If you want to experiment with Upstart, or have need of it due to a need for upstart jobs, this element will build the image with upstart as the init system. Note that this works with the ``debian`` element, not the ``debian-minimal`` element. .. element_deps:: diskimage-builder-2.11.0/diskimage_builder/elements/debian-upstart/element-deps000066400000000000000000000000071324273057700276710ustar00rootroot00000000000000debian diskimage-builder-2.11.0/diskimage_builder/elements/debian-upstart/root.d/000077500000000000000000000000001324273057700265745ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian-upstart/root.d/05-debian-upstart000077500000000000000000000002611324273057700316650ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail cat > ${TARGET_ROOT}/.extra_settings << EOF DIB_DEBIAN_ALT_INIT_PACKAGE=upstart EOF diskimage-builder-2.11.0/diskimage_builder/elements/debian-upstart/root.d/20-debian-fix-upstart-jobs000077500000000000000000000013101324273057700333750ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail # Working around bug in Debian cloud-init packages with upstart # where startpar will wait forever for these because they are tasks # http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=735204 for sysv_script in cloud-init cloud-init-local cloud-config cloud-final ; do sudo rm -f ${TARGET_ROOT}/etc/rc2.d/??${sysv_script} ${TARGET_ROOT}/etc/init.d/${sysv_script} done # And working around upstart job bug in Debian where cloud-config will never run # http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=735207 sudo sed -i -e 's/start on .*/start on started rc RUNLEVEL=[2345]/' ${TARGET_ROOT}/etc/init/cloud-config.conf diskimage-builder-2.11.0/diskimage_builder/elements/debian/000077500000000000000000000000001324273057700236675ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian/README.rst000066400000000000000000000007451324273057700253640ustar00rootroot00000000000000====== debian ====== This element is based on ``debian-minimal`` with ``cloud-init`` and related tools installed. This produces something more like a standard upstream cloud image. By default this element creates the latest stable release. The exact setting can be found in the ``debian-minimal/environment.d`` directory in the variable ``DIB_RELEASE``. If a different release of Debian should be created, the variable ``DIB_RELEASE`` can be set appropriately. .. element_deps:: diskimage-builder-2.11.0/diskimage_builder/elements/debian/element-deps000066400000000000000000000000461324273057700261740ustar00rootroot00000000000000debian-minimal openssh-server sysprep diskimage-builder-2.11.0/diskimage_builder/elements/debian/element-provides000066400000000000000000000000211324273057700270650ustar00rootroot00000000000000operating-system diskimage-builder-2.11.0/diskimage_builder/elements/debian/install.d/000077500000000000000000000000001324273057700255575ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian/install.d/10-cloud-opinions000077500000000000000000000026361324273057700306740ustar00rootroot00000000000000#!/bin/bash # # Copyright 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail DIB_DEBIAN_ALT_INIT_PACKAGE=${DIB_DEBIAN_ALT_INIT_PACKAGE:-sysvinit} adduser --gecos Debian-cloud-init-user --disabled-password --quiet debian install -d -m 0755 -o root -g root /etc/sudoers.d echo 'debian ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/debian-cloud-init chmod 0440 /etc/sudoers.d/debian-cloud-init if [ "$DIB_DEBIAN_ALT_INIT_PACKAGE" != "sysvinit" ]; then # To avoid a conflict against an essential package, we need to remove sysvinit first dpkg --purge --force remove-essential sysvinit apt-get install -y $apt_get_bp_extra_opts $DIB_DEBIAN_ALT_INIT_PACKAGE printf \"Package: sysvinit\nPin: origin ""\nPin-Priority: -1\n\" > \ /etc/apt/preferences.d/sysvinit > /etc/apt/preferences.d/sysvinit fi diskimage-builder-2.11.0/diskimage_builder/elements/debian/package-installs.yaml000066400000000000000000000002061324273057700277730ustar00rootroot00000000000000file: less: kbd: curl: rsync: bash-completion: isc-dhcp-client: net-tools: cloud-init: cloud-utils: cloud-initramfs-growroot: dialog: diskimage-builder-2.11.0/diskimage_builder/elements/debian/test-elements/000077500000000000000000000000001324273057700264605ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian/test-elements/build-succeeds/000077500000000000000000000000001324273057700313535ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian/test-elements/build-succeeds/README.rst000066400000000000000000000000641324273057700330420ustar00rootroot00000000000000Test that we can successfully build a debian image. diskimage-builder-2.11.0/diskimage_builder/elements/debian/test-elements/build-succeeds/element-deps000066400000000000000000000000051324273057700336530ustar00rootroot00000000000000base environment.d/000077500000000000000000000000001324273057700340625ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian/test-elements/build-succeeds09-debian-stable000066400000000000000000000002541324273057700367260ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debian/test-elements/build-succeeds/environment.d# We don't want to build against Debian unstable # (Needs to be a 09- because must be executed before # debian-minimal 10-debian-minimal.bash.) export DIB_RELEASE="stable" diskimage-builder-2.11.0/diskimage_builder/elements/debootstrap/000077500000000000000000000000001324273057700247735ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debootstrap/README.rst000066400000000000000000000057661324273057700265000ustar00rootroot00000000000000=========== debootstrap =========== Base element for creating minimal debian-based images. This element is incomplete by itself, you'll want to use elements like debian-minimal or ubuntu-minimal to get an actual base image. There are two ways to configure apt-sources: 1. Using the standard way of defining the default, backports, updates and security repositories is the default. In this case you can overwrite the two environment variables to adapt the behavior: * ``DIB_DISTRIBUTION_MIRROR``: the mirror to use (default: ``__) * ``DIB_DEBIAN_COMPONENTS``: (default: ``main``) a comma separated list of components. For Debian this can be e.g. ``main,contrib,non-free``. By default only the ``main`` component is used. If ``DIB_DEBIAN_COMPONENTS`` (comma separated) from the ``debootstrap`` element has been set, that list of components will be used instead. Backports, updates and security are included unless ``DIB_RELEASE`` is ``unstable``. 2. Complete configuration given in the variable ``DIB_APT_SOURCES_CONF``. Each line contains exactly one entry for the sources.list.d directory. The first word must be the logical name (which is used as file name with ``.list`` automatically appended), followed by a colon ``:``, followed by the complete repository specification. .. code-block:: bash DIB_APT_SOURCES_CONF=\ "default:deb http://10.0.0.10/ stretch main contrib mysecurity:deb http://10.0.0.10/ stretch-security main contrib" If necessary, a custom apt keyring and debootstrap script can be supplied to the ``debootstrap`` command via ``DIB_APT_KEYRING`` and ``DIB_DEBIAN_DEBOOTSTRAP_SCRIPT`` respectively. Both options require the use of absolute rather than relative paths. Use of this element will also require the tool 'debootstrap' to be available on your system. It should be available on Ubuntu, Debian, and Fedora. It is also recommended that the 'debian-keyring' package be installed. The ``DIB_OFFLINE`` or more specific ``DIB_DEBIAN_USE_DEBOOTSTRAP_CACHE`` variables can be set to prefer the use of a pre-cached root filesystem tarball. The ``DIB_DEBOOTSTRAP_EXTRA_ARGS`` environment variable may be used to pass extra arguments to the debootstrap command used to create the base filesystem image. If --keyring is is used in ``DIB_DEBOOTSTRAP_EXTRA_ARGS``, it will override ``DIB_APT_KEYRING`` if that is used as well. For further information about ``DIB_DEBIAN_DEBOOTSTRAP_SCRIPT`` , ``DIB_DEBIAN_USE_DEBOOTSTRAP_CACHE`` and ``DIB_DEBOOTSTRAP_EXTRA_ARGS`` please consult "README.rst" of the debootstrap element. ------------------- Note on ARM systems ------------------- Because there is not a one-to-one mapping of ``ARCH`` to a kernel package, if you are building an image for ARM on debian, you need to specify which kernel you want in the environment variable ``DIB_ARM_KERNEL``. For instance, if you want the ``linux-image-mx5`` package installed, set ``DIB_ARM_KERNEL`` to ``mx5``. .. element_deps:: diskimage-builder-2.11.0/diskimage_builder/elements/debootstrap/cleanup.d/000077500000000000000000000000001324273057700266445ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debootstrap/cleanup.d/99-clean-up-cache000077500000000000000000000014511324273057700315770ustar00rootroot00000000000000#!/bin/bash # # Copyright 2016 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail # NOTE(pabelanger): Remove cached info about APT repositories, as it will go # stale fast. sudo rm -rf $TARGET_ROOT/var/lib/apt/lists/* diskimage-builder-2.11.0/diskimage_builder/elements/debootstrap/element-deps000066400000000000000000000000151324273057700272740ustar00rootroot00000000000000dpkg pkg-map diskimage-builder-2.11.0/diskimage_builder/elements/debootstrap/environment.d/000077500000000000000000000000001324273057700275615ustar00rootroot0000000000000010-debootstrap-default-locale000066400000000000000000000001211324273057700350420ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debootstrap/environment.dexport DIB_DEBOOTSTRAP_DEFAULT_LOCALE=${DIB_DEBOOTSTRAP_DEFAULT_LOCALE:-C.UTF-8} diskimage-builder-2.11.0/diskimage_builder/elements/debootstrap/install.d/000077500000000000000000000000001324273057700266635ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debootstrap/install.d/10-debian-networking000077500000000000000000000023761324273057700324460ustar00rootroot00000000000000#!/bin/bash # # Copyright 2015 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail # It would be eversogreat if we didn't need to do crap like this echo $DISTRO_NAME > /etc/hostname # cloud images expect eth0 and eth1 to use dhcp. mkdir -p /etc/network/interfaces.d if ! grep -E -q '^source(|-directory) /etc/network/interfaces.d/\*' /etc/network/interfaces; then echo "source /etc/network/interfaces.d/*" >> /etc/network/interfaces echo 'Network configuration set to source /etc/network/interfaces.d/*' fi for interface in eth0 eth1; do cat << EOF | tee /etc/network/interfaces.d/$interface auto $interface iface $interface inet dhcp EOF done diskimage-builder-2.11.0/diskimage_builder/elements/debootstrap/install.d/12-debian-locale-gen000077500000000000000000000021571324273057700322640ustar00rootroot00000000000000#!/bin/bash # # Copyright 2015 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail [ -f /etc/locale.gen ] && rm /etc/locale.gen debconf-set-selections - < /etc/default/locale < /etc/modprobe.d/blacklist.conf diskimage-builder-2.11.0/diskimage_builder/elements/debootstrap/package-installs.yaml000066400000000000000000000001071324273057700310770ustar00rootroot00000000000000adduser: ca-certificates: locales: lsb-release: phase: pre-install.d diskimage-builder-2.11.0/diskimage_builder/elements/debootstrap/pkg-map000066400000000000000000000000721324273057700262510ustar00rootroot00000000000000{ "default": { "cloud-initramfs-growroot": "" } } diskimage-builder-2.11.0/diskimage_builder/elements/debootstrap/root.d/000077500000000000000000000000001324273057700262005ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/debootstrap/root.d/08-debootstrap000077500000000000000000000054431324273057700307070ustar00rootroot00000000000000#!/bin/bash # # Copyright 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail if [ -f ${TARGET_ROOT}/.extra_settings ] ; then . ${TARGET_ROOT}/.extra_settings fi [ -n "$DISTRO_NAME" ] [ -n "$DIB_RELEASE" ] [ -n "$DIB_DISTRIBUTION_MIRROR" ] DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main} DIB_DEBOOTSTRAP_EXTRA_ARGS=${DIB_DEBOOTSTRAP_EXTRA_ARGS:-} DEBOOTSTRAP_TARBALL=$DIB_IMAGE_CACHE/debootstrap-${DISTRO_NAME}-${DIB_RELEASE}-${ARCH}.tar.gz http_proxy=${http_proxy:-} no_proxy=${no_proxy:-} DEBOOTSTRAP_QEMU="" if [ -n "$DIB_OFFLINE" -o -n "${DIB_DEBIAN_USE_DEBOOTSTRAP_CACHE:-}" ] && [ -f $DEBOOTSTRAP_TARBALL ] ; then echo $DEBOOTSTRAP_TARBALL found in cache. Using. sudo tar -C $TARGET_ROOT --numeric-owner -xzf $DEBOOTSTRAP_TARBALL else KEYRING_OPT= if [ -n "${DIB_APT_KEYRING:-${DIB_DEBIAN_KEYRING:-}}" ] ; then KEYRING_OPT="--keyring=${DIB_APT_KEYRING:-${DIB_DEBIAN_KEYRING:-}}" fi # Have to --include=python because of dib-run-parts # Have to --include=sudo for pre-install.d use of sudoers files # Have to --include=busybox because initramfs needs it if [ $(dpkg --print-architecture) != ${ARCH} ]; then # Target architecture diffrent from host architecture: # there is the need to call 'qemu-debootstrap'. DEBOOTSTRAP_QEMU="qemu-" fi sudo sh -c "http_proxy=$http_proxy no_proxy=$no_proxy ${DEBOOTSTRAP_QEMU}debootstrap --verbose \ --variant=minbase \ --components=${DIB_DEBIAN_COMPONENTS} \ --arch=${ARCH} \ $KEYRING_OPT \ $DIB_DEBOOTSTRAP_EXTRA_ARGS \ $DIB_RELEASE \ $TARGET_ROOT \ $DIB_DISTRIBUTION_MIRROR \ ${DIB_DEBIAN_DEBOOTSTRAP_SCRIPT:-}" # debootstrap creates sometimes a not-usable # /etc/apt/sources.list: DIB wants to set up its own anyway # and the old has to go. sudo rm -fr ${TARGET_ROOT}/etc/apt/sources.list \ ${TARGET_ROOT}/etc/apt/sources.list.d echo Caching debootstrap result in $DEBOOTSTRAP_TARBALL if [ "${DIB_DEBOOTSTRAP_CACHE:-0}" != "0" ]; then sudo tar --numeric-owner -C $TARGET_ROOT -zcf $DEBOOTSTRAP_TARBALL --exclude='./tmp/*' . fi fi sudo rm -f ${TARGET_ROOT}/.extra_settings diskimage-builder-2.11.0/diskimage_builder/elements/deploy-baremetal/000077500000000000000000000000001324273057700256735ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-baremetal/README.rst000066400000000000000000000002621324273057700273620ustar00rootroot00000000000000================ deploy-baremetal ================ A ramdisk that will expose the machine primary disk over iSCSI and reboot once baremetal-deploy-helper signals it is finished. diskimage-builder-2.11.0/diskimage_builder/elements/deploy-baremetal/binary-deps.d/000077500000000000000000000000001324273057700303325ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-baremetal/binary-deps.d/deploy000066400000000000000000000000211324273057700315420ustar00rootroot00000000000000bash tgtd tgtadm diskimage-builder-2.11.0/diskimage_builder/elements/deploy-baremetal/element-deps000066400000000000000000000000211324273057700301710ustar00rootroot00000000000000package-installs diskimage-builder-2.11.0/diskimage_builder/elements/deploy-baremetal/init.d/000077500000000000000000000000001324273057700270605ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-baremetal/init.d/80-deploy000066400000000000000000000014351324273057700305270ustar00rootroot00000000000000if [ -z "$ISCSI_TARGET_IQN" ]; then err_msg "iscsi_target_iqn is not defined" troubleshoot fi t=0 while ! target_disk=$(find_disk "$DISK"); do if [ $t -eq 10 ]; then break fi t=$(($t + 1)) sleep 1 done if [ -z "$target_disk" ]; then err_msg "Could not find disk to use." troubleshoot fi echo "start iSCSI target on $target_disk" start_iscsi_target "$ISCSI_TARGET_IQN" "$target_disk" ALL if [ $? -ne 0 ]; then err_msg "Failed to start iscsi target." troubleshoot fi echo "request boot server to deploy image" d="i=$DEPLOYMENT_ID&k=$DEPLOYMENT_KEY&a=$BOOT_IP_ADDRESS&n=$ISCSI_TARGET_IQN&e=$FIRST_ERR_MSG" wget --post-data "$d" "http://$BOOT_SERVER:10000" echo "waiting for notice of complete" nc -l -p 10000 echo "stop iSCSI target on $target_disk" stop_iscsi_target diskimage-builder-2.11.0/diskimage_builder/elements/deploy-baremetal/package-installs.yaml000066400000000000000000000000051324273057700317740ustar00rootroot00000000000000tgt: diskimage-builder-2.11.0/diskimage_builder/elements/deploy-kexec/000077500000000000000000000000001324273057700250365ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-kexec/README.rst000066400000000000000000000003031324273057700265210ustar00rootroot00000000000000============ deploy-kexec ============ Boots into the new image once baremetal-deploy-helper signals it is finished by downloading the kernel and ramdisk via tftp, and using the kexec utilities. diskimage-builder-2.11.0/diskimage_builder/elements/deploy-kexec/binary-deps.d/000077500000000000000000000000001324273057700274755ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-kexec/binary-deps.d/deploy-kexec000066400000000000000000000000061324273057700320050ustar00rootroot00000000000000kexec diskimage-builder-2.11.0/diskimage_builder/elements/deploy-kexec/element-deps000066400000000000000000000000301324273057700273340ustar00rootroot00000000000000deploy package-installs diskimage-builder-2.11.0/diskimage_builder/elements/deploy-kexec/init.d/000077500000000000000000000000001324273057700262235ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-kexec/init.d/81-deploy-kexec000066400000000000000000000007131324273057700307660ustar00rootroot00000000000000TFTP_MAC_ADDRESS="01-"`echo $BOOT_MAC_ADDRESS | sed 's/:/-/g' | tr "A-Z" "a-z"` tftp -r /tftpboot/pxelinux.cfg/${TFTP_MAC_ADDRESS} -g ${BOOT_SERVER} APPEND=$(tail -n1 ${TFTP_MAC_ADDRESS} | cut -f3- -d" ") KERNEL=$(get_kernel_parameter BOOT_IMAGE | sed 's/deploy_//g') RAMDISK=$(echo $KERNEL | sed 's/kernel/ramdisk/') tftp -r ${KERNEL} -g ${BOOT_SERVER} tftp -r ${RAMDISK} -g ${BOOT_SERVER} kexec -l kernel --append="${APPEND}" --initrd=ramdisk && kexec -e diskimage-builder-2.11.0/diskimage_builder/elements/deploy-kexec/package-installs.yaml000066400000000000000000000000151324273057700311400ustar00rootroot00000000000000kexec-tools: diskimage-builder-2.11.0/diskimage_builder/elements/deploy-targetcli/000077500000000000000000000000001324273057700257155ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-targetcli/README.rst000066400000000000000000000004171324273057700274060ustar00rootroot00000000000000deploy-targetcli ================ Use targetcli for the deploy ramdisk Provides the necessary scripts and dependencies to use targetcli for exporting the iscsi target in the deploy ramdisk. Implemented as a dracut module, so will only work with dracut-based ramdisks. diskimage-builder-2.11.0/diskimage_builder/elements/deploy-targetcli/binary-deps.d/000077500000000000000000000000001324273057700303545ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-targetcli/binary-deps.d/deploy-targetcli000066400000000000000000000000121324273057700335400ustar00rootroot00000000000000targetcli diskimage-builder-2.11.0/diskimage_builder/elements/deploy-targetcli/dracut-drivers.d/000077500000000000000000000000001324273057700310755ustar00rootroot00000000000000targetcli-drivers000066400000000000000000000001401324273057700343660ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-targetcli/dracut-drivers.dtarget_core_mod iscsi_target_mod target_core_iblock target_core_file target_core_pscsi configfs diskimage-builder-2.11.0/diskimage_builder/elements/deploy-targetcli/element-deps000066400000000000000000000000211324273057700302130ustar00rootroot00000000000000package-installs diskimage-builder-2.11.0/diskimage_builder/elements/deploy-targetcli/extra-data.d/000077500000000000000000000000001324273057700301715ustar00rootroot0000000000000050-add-targetcli-module000077500000000000000000000004011324273057700342440ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-targetcli/extra-data.d#!/bin/bash set -eu set -o pipefail if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi MODULE_PATH="/usr/lib/dracut/modules.d" sudo mkdir -p ${TMP_MOUNT_PATH}${MODULE_PATH} sudo cp -r $(dirname $0)/module/ ${TMP_MOUNT_PATH}${MODULE_PATH}/50targetcli diskimage-builder-2.11.0/diskimage_builder/elements/deploy-targetcli/extra-data.d/module/000077500000000000000000000000001324273057700314565ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-targetcli/extra-data.d/module/iscsi-func000077500000000000000000000013451324273057700334520ustar00rootroot00000000000000function start_iscsi_target() { local iqn=$1 local dev=$2 local cli=$3 # used by tgtd mkdir -p /var/run ln -s /usr/bin/targetcli /targetcli_bin.py /targetcli-wrapper /backstores/block create block1 dev=$dev /targetcli-wrapper /iscsi create $iqn /targetcli-wrapper /iscsi/$iqn/tpg1/luns create /backstores/block/block1 1 /targetcli-wrapper /iscsi/$iqn/tpg1/portals create $BOOT_IP_ADDRESS /targetcli-wrapper /iscsi/$iqn/tpg1 set attribute authentication=0 /targetcli-wrapper /iscsi/$iqn/tpg1 set attribute demo_mode_write_protect=0 /targetcli-wrapper /iscsi/$iqn/tpg1 set attribute generate_node_acls=1 } function stop_iscsi_target() { /targetcli-wrapper clearconfig confirm=True } module-setup.sh000077500000000000000000000014021324273057700343560ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-targetcli/extra-data.d/module#!/bin/bash # Dracut is bash too, and it doesn't play nicely with our usual sets # dib-lint: disable=setu sete setpipefail dibdebugtrace check() { return 0 } depends() { return 0 } install() { inst /bin/targetcli inst "$moddir/targetcli-wrapper" /targetcli-wrapper inst "$moddir/iscsi-func" /iscsi-func # Install all of Python # TODO(bnemec): At some point this will need to be extended to support # Python 3, but for the moment we aren't using that anyway. inst /usr/bin/python local all_of_python=() while IFS='' read -r -d '' i; do all_of_python+=("$i") done < <(find /usr/lib64/python2.7/ /usr/lib/python2.7/ -type f -not -name "*.pyc" -not -name "*.pyo" -print0) inst_multiple "${all_of_python[@]}" } targetcli-wrapper000077500000000000000000000016171324273057700347660ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-targetcli/extra-data.d/module#!/usr/bin/python # Copyright 2014 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import rtslib import targetcli_bin class MyISCSIFabricModule(rtslib.fabric.ISCSIFabricModule): def __init__(self): super(rtslib.fabric.ISCSIFabricModule, self).__init__('iscsi') self.wwn_types = ('free',) rtslib.fabric.fabric_modules['iscsi'] = MyISCSIFabricModule targetcli_bin.main() diskimage-builder-2.11.0/diskimage_builder/elements/deploy-targetcli/package-installs.yaml000066400000000000000000000000131324273057700320150ustar00rootroot00000000000000targetcli: diskimage-builder-2.11.0/diskimage_builder/elements/deploy-tgtadm/000077500000000000000000000000001324273057700252175ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-tgtadm/README.rst000066400000000000000000000003761324273057700267140ustar00rootroot00000000000000deploy-tgtadm ============= Use tgtadm and tgtd for the deploy ramdisk Provides the necessary scripts and dependencies to use tgtadm and tgtd for exporting the iscsi target in the deploy ramdisk. Will only work with the standard (not dracut) ramdisk. diskimage-builder-2.11.0/diskimage_builder/elements/deploy-tgtadm/binary-deps.d/000077500000000000000000000000001324273057700276565ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-tgtadm/binary-deps.d/deploy-tgtadm000066400000000000000000000000141324273057700323460ustar00rootroot00000000000000tgtadm tgtd diskimage-builder-2.11.0/diskimage_builder/elements/deploy-tgtadm/extra-data.d/000077500000000000000000000000001324273057700274735ustar00rootroot0000000000000050-inject-tgtadm-iscsi-func000077500000000000000000000003731324273057700343620ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-tgtadm/extra-data.d#!/bin/bash set -eu set -o pipefail if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi RAMDISK_SCRIPT_PATH="$TMP_MOUNT_PATH/tmp/ramdisk-build/scripts/d" mkdir -p ${RAMDISK_SCRIPT_PATH} cp $(dirname $0)/scripts/iscsi-func ${RAMDISK_SCRIPT_PATH} diskimage-builder-2.11.0/diskimage_builder/elements/deploy-tgtadm/extra-data.d/scripts/000077500000000000000000000000001324273057700311625ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/deploy-tgtadm/extra-data.d/scripts/iscsi-func000066400000000000000000000011301324273057700331430ustar00rootroot00000000000000function start_iscsi_target() { local iqn=$1 local dev=$2 local cli=$3 # used by tgtd mkdir -p /var/run tgtd wait_for 10 0.5 check_tgtd_socket tgtadm --lld iscsi --mode target --op new --tid 1 --targetname "$iqn" tgtadm --lld iscsi --mode logicalunit --op new --tid 1 --lun 1 --backing-store "$dev" tgtadm --lld iscsi --mode target --op bind --tid 1 --initiator-address "$cli" } function stop_iscsi_target() { tgtadm --lld iscsi --mode logicalunit --op delete --tid 1 --lun 1 tgtadm --lld iscsi --mode target --op delete --tid 1 killall tgtd } diskimage-builder-2.11.0/diskimage_builder/elements/deploy-tgtadm/package-installs.yaml000066400000000000000000000000051324273057700313200ustar00rootroot00000000000000tgt: diskimage-builder-2.11.0/diskimage_builder/elements/devuser/000077500000000000000000000000001324273057700241225ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/devuser/README.rst000066400000000000000000000023461324273057700256160ustar00rootroot00000000000000======= devuser ======= Creates a user that is useful for development / debugging. The following environment variables can be useful for configuration: Environment Variables --------------------- DIB_DEV_USER_USERNAME :Required: No :Default: devuser :Description: Username for the created user. DIB_DEV_USER_SHELL :Required: No :Default: System default (The useradd default is used) :Description: Full path for the shell of the user. This is passed to useradd using the -s parameter. Note that this does not install the (possibly) required shell package. DIB_DEV_USER_PWDLESS_SUDO :Required: No :Default: No :Description: Enable passwordless sudo for the user. DIB_DEV_USER_AUTHORIZED_KEYS :Required: No :Default: $HOME/.ssh/id_{rsa,dsa}.pub :Description: Path to a file to copy into this users' .ssh/authorized_keys If this is not specified then an attempt is made to use a the building user's public key. To disable this behavior specify an invalid path for this variable (such as /dev/null). DIB_DEV_USER_PASSWORD :Required: No :Default: Password is disabled :Description: Set the default password for this user. This is a fairly insecure method of setting the password and is not advised. diskimage-builder-2.11.0/diskimage_builder/elements/devuser/environment.d/000077500000000000000000000000001324273057700267105ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/devuser/environment.d/50-devuser000066400000000000000000000004571324273057700305400ustar00rootroot00000000000000export DIB_DEV_USER_USERNAME=${DIB_DEV_USER_USERNAME:-devuser} export DIB_DEV_USER_SHELL=${DIB_DEV_USER_SHELL:-} export DIB_DEV_USER_PWDLESS_SUDO=${DIB_DEV_USER_PWDLESS_SUDO:-} export DIB_DEV_USER_AUTHORIZED_KEYS=${DIB_DEV_USER_AUTHORIZED_KEYS:-} export DIB_DEV_USER_PASSWORD=${DIB_DEV_USER_PASSWORD:-} diskimage-builder-2.11.0/diskimage_builder/elements/devuser/extra-data.d/000077500000000000000000000000001324273057700263765ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/devuser/extra-data.d/50-devuser000077500000000000000000000007611324273057700302270ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail if [ -n "$DIB_DEV_USER_AUTHORIZED_KEYS" ]; then if [ -f "$DIB_DEV_USER_AUTHORIZED_KEYS" ]; then cat $DIB_DEV_USER_AUTHORIZED_KEYS >> $TMP_HOOKS_PATH/devuser-ssh-authorized-keys fi else for fmt in rsa dsa; do if [ -f "$HOME/.ssh/id_$fmt.pub" ]; then cat $HOME/.ssh/id_$fmt.pub >> $TMP_HOOKS_PATH/devuser-ssh-authorized-keys break fi done fi diskimage-builder-2.11.0/diskimage_builder/elements/devuser/install.d/000077500000000000000000000000001324273057700260125ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/devuser/install.d/50-devuser000077500000000000000000000017461324273057700276470ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail user_shell_args= if [ -n "${DIB_DEV_USER_SHELL}" ]; then user_shell_args="-s ${DIB_DEV_USER_SHELL}" fi useradd -m ${DIB_DEV_USER_USERNAME} $user_shell_args set +x if [ -n "${DIB_DEV_USER_PASSWORD}" ]; then echo "Setting password." echo "${DIB_DEV_USER_USERNAME}:${DIB_DEV_USER_PASSWORD}" | chpasswd fi set -x if [ -n "${DIB_DEV_USER_PWDLESS_SUDO}" ]; then cat > /etc/sudoers.d/${DIB_DEV_USER_USERNAME} << EOF ${DIB_DEV_USER_USERNAME} ALL=(ALL) NOPASSWD:ALL EOF chmod 0440 /etc/sudoers.d/${DIB_DEV_USER_USERNAME} visudo -c || rm /etc/sudoers.d/${DIB_DEV_USER_USERNAME} fi if [ -f /tmp/in_target.d/devuser-ssh-authorized-keys ]; then mkdir -p /home/${DIB_DEV_USER_USERNAME}/.ssh cp /tmp/in_target.d/devuser-ssh-authorized-keys /home/${DIB_DEV_USER_USERNAME}/.ssh/authorized_keys fi chown -R ${DIB_DEV_USER_USERNAME}:${DIB_DEV_USER_USERNAME} /home/${DIB_DEV_USER_USERNAME} diskimage-builder-2.11.0/diskimage_builder/elements/dhcp-all-interfaces/000077500000000000000000000000001324273057700262525ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dhcp-all-interfaces/README.rst000066400000000000000000000021141324273057700277370ustar00rootroot00000000000000=================== dhcp-all-interfaces =================== Autodetect network interfaces during boot and configure them for DHCP The rationale for this is that we are likely to require multiple network interfaces for use cases such as baremetal and there is no way to know ahead of time which one is which, so we will simply run a DHCP client on all interfaces with real MAC addresses (except lo) that are visible on the first boot. On non-Gentoo based distributions the script /usr/local/sbin/dhcp-all-interfaces.sh will be called early in each boot and will scan available network interfaces and ensure they are configured properly before networking services are started. On Gentoo based distributions we will install the dhcpcd package and ensure the service starts at boot. This service automatically sets up all interfaces found via dhcp and/or dhcpv6 (or SLAAC). Environment Variables --------------------- DIB_DHCP_TIMEOUT :Required: No :Default: 30 :Description: Amount of time in seconds that the systemd service will wait to get an address. :Example: DIB_DHCP_TIMEOUT=300 diskimage-builder-2.11.0/diskimage_builder/elements/dhcp-all-interfaces/element-deps000066400000000000000000000000511324273057700305530ustar00rootroot00000000000000dib-init-system package-installs pkg-map diskimage-builder-2.11.0/diskimage_builder/elements/dhcp-all-interfaces/install.d/000077500000000000000000000000001324273057700301425ustar00rootroot0000000000000050-dhcp-all-interfaces000077500000000000000000000032751324273057700340470ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dhcp-all-interfaces/install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail SCRIPTDIR=$(dirname $0) # this script is not needed on Gentoo. if [ "$DISTRO_NAME" != "gentoo" ]; then install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-all-interfaces.sh /usr/local/sbin/dhcp-all-interfaces.sh fi if [ "$DIB_INIT_SYSTEM" == "upstart" ]; then if [ -e "/etc/redhat-release" ] ; then # the init system is upstart but networking is using sysv compatibility (i.e. Centos/RHEL 6) install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-all-interfaces.init /etc/init.d/dhcp-all-interfaces chkconfig dhcp-all-interfaces on else install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-all-interfaces.conf /etc/init/dhcp-all-interfaces.conf fi elif [ "$DIB_INIT_SYSTEM" == "systemd" ]; then install -D -g root -o root -m 0644 ${SCRIPTDIR}/dhcp-interface@.service /usr/lib/systemd/system/dhcp-interface@.service install -D -g root -o root -m 0644 ${SCRIPTDIR}/dhcp-all-interfaces-udev.rules /etc/udev/rules.d/99-dhcp-all-interfaces.rules sed -i "s/TimeoutStartSec=DIB_DHCP_TIMEOUT/TimeoutStartSec=${DIB_DHCP_TIMEOUT:-30}s/" /usr/lib/systemd/system/dhcp-interface@.service elif [ "$DIB_INIT_SYSTEM" == "sysv" ]; then install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-all-interfaces.init /etc/init.d/dhcp-all-interfaces update-rc.d dhcp-all-interfaces defaults elif [ "$DISTRO_NAME" == "gentoo" ]; then # let ipv6 use normal slaac sed -i 's/slaac/#slaac/g' /etc/dhcpcd.conf # don't let dhcpcd set domain name or hostname sed -i 's/domain_name\,\ domain_search\,\ host_name/domain_search/g' /etc/dhcpcd.conf rc-update add dhcpcd default fi 60-remove-cloud-image-interfaces000077500000000000000000000012141324273057700360340ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dhcp-all-interfaces/install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # Cloud images may hard code the eth0 interfaces so they # boot with DHCP. # Fedora rm -f /etc/sysconfig/network-scripts/ifcfg-eth* # Ubuntu rm -f /etc/network/interfaces.d/eth*.cfg # Debian rm -f /etc/network/interfaces.d/eth* # Gentoo rm -f /etc/conf.d/net* # /etc/network/interfaces distributions if [ -f "/etc/network/interfaces" ]; then printf "auto lo\niface lo inet loopback\n\n" > /etc/network/interfaces if [ -d "/etc/network/interfaces.d/" ]; then printf "source-directory interfaces.d\n\n" >> /etc/network/interfaces fi fi dhcp-all-interfaces-udev.rules000066400000000000000000000001621324273057700357040ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dhcp-all-interfaces/install.dSUBSYSTEM=="net", KERNEL!="lo", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}+="dhcp-interface@$name.service" dhcp-all-interfaces.conf000066400000000000000000000005321324273057700345370ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dhcp-all-interfaces/install.d# Call a script to generate a /etc/network/interfaces file to DHCP all available interfaces # Then remove this config file so the script is never run again description "DHCP any connected, but unconfigured network interfaces" start on starting network-interface instance $INTERFACE task exec /usr/local/sbin/dhcp-all-interfaces.sh $INTERFACE dhcp-all-interfaces.init000077500000000000000000000012741324273057700345640ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dhcp-all-interfaces/install.d#!/bin/sh -e ### BEGIN INIT INFO # Provides: dhcp-all-interfaces # Required-Start: $local_fs # Required-Stop: $local_fs # Default-Start: S # Default-Stop: 0 6 # X-Start-Before: networking # chkconfig: 3 9 50 # Short-Description: Autodetect network interfaces # Description: Autodetect network interfaces during boot and configure them for DHCP ### END INIT INFO NAME=dhcp-all-interfaces INIT_NAME=/etc/init.d/${NAME} SCRIPT_NAME=/usr/local/sbin/${NAME}.sh [ -x $SCRIPT_NAME ] || exit 0 case "$1" in start) $SCRIPT_NAME ;; stop) ;; *) echo "Usage: $INIT_NAME {start|stop}" exit 1 ;; esac exit 0 dhcp-all-interfaces.sh000077500000000000000000000065611324273057700342370ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dhcp-all-interfaces/install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail INTERFACE=${1:-} #optional, if not specified configure all available interfaces ENI_FILE="/etc/network/interfaces" PATH=/sbin:$PATH if [ -d "/etc/network" ]; then CONF_TYPE="eni" elif [ -d "/etc/sysconfig/network-scripts/" ]; then CONF_TYPE="rhel-netscripts" SCRIPTS_PATH="/etc/sysconfig/network-scripts/" elif [ -d "/etc/sysconfig/network/" ]; then # SUSE network scripts location CONF_TYPE="suse-netscripts" SCRIPTS_PATH="/etc/sysconfig/network/" else echo "Unsupported network configuration type!" exit 1 fi ARGS="$0 $@" function serialize_me() { if [ "$CONF_TYPE" == "eni" ]; then # Serialize runs so that we don't miss hot-add interfaces FLOCKED=${FLOCKED:-} if [ -z "$FLOCKED" ] ; then FLOCKED=true exec flock -x $ENI_FILE $ARGS fi fi } function get_if_link() { cat /sys/class/net/${1}/carrier || echo 0 } function get_if_type() { cat /sys/class/net/${1}/type } function enable_interface() { local interface=$1 serialize_me if [ "$CONF_TYPE" == "eni" ]; then printf "auto $interface\niface $interface inet dhcp\n\n" >>$ENI_FILE elif [ "$CONF_TYPE" == "rhel-netscripts" ]; then if [ "$(get_if_type $interface)" == "32" ]; then printf "DEVICE=\"$interface\"\nBOOTPROTO=\"dhcp\"\nONBOOT=\"yes\"\nTYPE=\"InfiniBand\"\nCONNECTED_MODE=\"no\"\nDEFROUTE=\"yes\"\nPEERDNS=\"yes\"\nPEERROUTES=\"yes\"\nIPV4_FAILURE_FATAL=\"yes\"\nIPV6INIT=\"no\"" >"${SCRIPTS_PATH}ifcfg-$interface" else printf "DEVICE=\"$interface\"\nBOOTPROTO=\"dhcp\"\nONBOOT=\"yes\"\nTYPE=\"Ethernet\"" >"${SCRIPTS_PATH}ifcfg-$interface" fi elif [ "$CONF_TYPE" == "suse-netscripts" ]; then printf "BOOTPROTO=\"dhcp\"\nSTARTMODE=\"auto\"" >"${SCRIPTS_PATH}ifcfg-$interface" fi echo "Configured $1" } function config_exists() { local interface=$1 if [[ "$CONF_TYPE" =~ "netscripts" ]]; then if [ -f "${SCRIPTS_PATH}ifcfg-$interface" ]; then return 0 fi else if ifquery $interface >/dev/null 2>&1; then if [ -z "$(ifquery $interface 2>&1)" ]; then return 1 else return 0 fi else return 1 fi fi return 1 } function inspect_interface() { local interface=$1 local mac_addr_type mac_addr_type=$(cat /sys/class/net/${interface}/addr_assign_type) echo -n "Inspecting interface: $interface..." if config_exists $interface; then echo "Has config, skipping." elif [ "$mac_addr_type" != "0" ]; then echo "Device has generated MAC, skipping." else local has_link local tries for ((tries = 0; tries < 20; tries++)); do # Need to set the link up on each iteration ip link set dev $interface up &>/dev/null has_link=$(get_if_link $interface) [ "$has_link" == "1" ] && break sleep 1 done if [ "$has_link" == "1" ]; then enable_interface "$interface" else echo "No link detected, skipping" fi fi } if [ -n "$INTERFACE" ]; then inspect_interface $INTERFACE else for iface in $(ls /sys/class/net | grep -v ^lo$); do inspect_interface $iface done fi dhcp-interface@.service000066400000000000000000000013621324273057700344230ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dhcp-all-interfaces/install.d[Unit] Description=DHCP interface %i # We want to run after network.target so it doesn't try to bring # up the interfaces a second time, but network-online should not # be reached until after we've brought up the interfaces. # We also need to break the default dependencies which prevents # this from operating on Ubuntu/Debian as the unit gets stuck # into a cyclical dependency loop. DefaultDependencies=no After=network.target Before=network-online.target Wants=network-online.target ConditionPathExists=!/etc/sysconfig/network-scripts/ifcfg-%i [Service] Type=oneshot User=root ExecStartPre=/usr/local/sbin/dhcp-all-interfaces.sh %i ExecStart=/sbin/ifup %i RemainAfterExit=true TimeoutStartSec=DIB_DHCP_TIMEOUT [Install] WantedBy=multi-user.target diskimage-builder-2.11.0/diskimage_builder/elements/dhcp-all-interfaces/package-installs.yaml000066400000000000000000000000271324273057700323570ustar00rootroot00000000000000dhcp-client: ifupdown: diskimage-builder-2.11.0/diskimage_builder/elements/dhcp-all-interfaces/pkg-map000066400000000000000000000005031324273057700275270ustar00rootroot00000000000000{ "family": { "redhat": { "dhcp-client": "dhclient" }, "gentoo": { "dhcp-client": "net-misc/dhcpcd" }, "suse": { "dhcp-client": "dhcp-client" }, "debian": { "ifupdown": "ifupdown" } }, "default": { "dhcp-client": "isc-dhcp-client", "ifupdown": "" } } diskimage-builder-2.11.0/diskimage_builder/elements/dib-init-system/000077500000000000000000000000001324273057700254665ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dib-init-system/README.rst000066400000000000000000000010621324273057700271540ustar00rootroot00000000000000=============== dib-init-system =============== Installs a script (dib-init-system) which outputs the type of init system in use on the target image. Also sets an environment variable ``DIB_INIT_SYSTEM`` to this value. Any files placed in a ``init-scripts/INIT_SYSTEM`` directory inside the element will be copied into the appropriate directory if ``INIT_SYSTEM`` is in use on the host. Environment Variables --------------------- DIB_INIT_SYSTEM :Description: One of upstart, systemd, or sysv depending on the init system in use for the target image. diskimage-builder-2.11.0/diskimage_builder/elements/dib-init-system/dib-init-system000077500000000000000000000012321324273057700304330ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # Debian Jessie and Debian Stretch use /bin/systemctl. # (/sbin/init is only available if systemd-sysv is installed.) if [ -f /usr/bin/systemctl -o -f /bin/systemctl ]; then echo "systemd" elif [[ -f /sbin/initctl ]]; then echo "upstart" elif [[ -f /etc/gentoo-release ]]; then if [[ "${GENTOO_PROFILE}" =~ systemd ]]; then echo "systemd" else echo "openrc" fi elif [[ -f /sbin/init ]]; then if [[ -f /bin/systemd ]]; then echo "systemd" else echo "sysv" fi else echo "Unknown init system" exit 1 fi diskimage-builder-2.11.0/diskimage_builder/elements/dib-init-system/environment.d/000077500000000000000000000000001324273057700302545ustar00rootroot0000000000000010-dib-init-system.bash000066400000000000000000000001421324273057700342700ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dib-init-system/environment.dDIB_INIT_SYSTEM=$(PATH="$PATH:$(dirname $BASH_SOURCE)/.." dib-init-system) export DIB_INIT_SYSTEM diskimage-builder-2.11.0/diskimage_builder/elements/dib-init-system/install.d/000077500000000000000000000000001324273057700273565ustar00rootroot0000000000000020-install-init-scripts000077500000000000000000000014151324273057700335410ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dib-init-system/install.d#!/bin/bash # Note that this relies on the detail that all elements share one dir inside # the chroot. This will copy all the files that elements have added to # element/static into the image. Mode, symlinks etc will be respected. if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail scripts_dir="$(dirname $0)/../init-scripts/$DIB_INIT_SYSTEM/" if [ -d "$scripts_dir" ]; then dest= case $DIB_INIT_SYSTEM in upstart) dest=/etc/init/ ;; openrc) dest=/etc/init.d/ ;; systemd) dest=/usr/lib/systemd/system/ ;; sysv) dest=/etc/init.d/ ;; esac if [ -z "$dest" ]; then echo "ERROR: DIB_INIT_SYSTEM ($DIB_INIT_SYSTEM) is not a known type" exit 1 fi cp -RP $scripts_dir. $dest || true fi diskimage-builder-2.11.0/diskimage_builder/elements/dib-init-system/post-install.d/000077500000000000000000000000001324273057700303415ustar00rootroot0000000000000010-enable-init-scripts000077500000000000000000000012421324273057700343010ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dib-init-system/post-install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail SCRIPTS_DIR="$(dirname $0)/../init-scripts/${DIB_INIT_SYSTEM}/" if [[ -d "${SCRIPTS_DIR}" ]]; then # figure out init prefix case "${DIB_INIT_SYSTEM}" in upstart) ;; openrc) # only gentoo needs manual runlevel adding for INIT_SCRIPT in "${SCRIPTS_DIR}"*; do rc-update add $(basename "${INIT_SCRIPT}") default done ;; systemd) ;; sysv) ;; *) echo "ERROR: DIB_INIT_SYSTEM (${DIB_INIT_SYSTEM}) is not a known type" exit 1 ;; esac fi diskimage-builder-2.11.0/diskimage_builder/elements/dib-init-system/pre-install.d/000077500000000000000000000000001324273057700301425ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dib-init-system/pre-install.d/04-dib-init-system000077500000000000000000000002501324273057700333270ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail install -m 0755 -o root -g root $(dirname $0)/../dib-init-system /usr/bin/ diskimage-builder-2.11.0/diskimage_builder/elements/dib-python/000077500000000000000000000000001324273057700245225ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dib-python/README.rst000066400000000000000000000017411324273057700262140ustar00rootroot00000000000000========== dib-python ========== Adds a symlink to `/usr/local/bin/dib-python` which points at either a `python2` or `python3` executable as appropriate. In-chroot scripts should use this as their interpreter (`#!/usr/local/bin/dib-python`) to make scripts that are compatible with both `python2` and `python3`. We can not assume `/usr/bin/python` exists, as some platforms have started shipping with only Python 3. `DIB_PYTHON` will be exported as the python interpreter. You should use this instead of `python script.py` (e.g. `${DIB_PYTHON} script.py`). Note you can also call `/usr/local/bin/dib-python script.py` but in some circumstances, such as creating a `virtualenv`, it can create somewhat confusing references to `dib-python` that remain in the built image. This does not install a python if one does not exist, and instead fails. This also exports a variable `DIB_PYTHON_VERSION` which will either be '2' or '3' depending on the python version which dib-python points to. diskimage-builder-2.11.0/diskimage_builder/elements/dib-python/environment.d/000077500000000000000000000000001324273057700273105ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dib-python/environment.d/50-dib-python-version000066400000000000000000000016361324273057700332230ustar00rootroot00000000000000# Pick which distros we need to force python2 if [ -z "${DIB_PYTHON_VERSION:-}" ]; then if [ "$DISTRO_NAME" == "ubuntu" ]; then if [ "$DIB_RELEASE" == "trusty" ]; then DIB_PYTHON_VERSION=2 fi elif [ "$DISTRO_NAME" == "debian" ]; then DIB_PYTHON_VERSION=2 elif [ "$DISTRO_NAME" == "fedora" ]; then if [ "$DIB_RELEASE" -le 22 ]; then DIB_PYTHON_VERSION=2 fi elif [ "$DISTRO_NAME" == "centos" ]; then DIB_PYTHON_VERSION=2 elif [ "$DISTRO_NAME" == "centos7" ]; then DIB_PYTHON_VERSION=2 elif [ "$DISTRO_NAME" == "rhel7" ]; then DIB_PYTHON_VERSION=2 elif [ "$DISTRO_NAME" == "opensuse" ]; then DIB_PYTHON_VERSION=2 fi fi if [ -z "${DIB_PYTHON_VERSION:-}" ]; then DIB_PYTHON_VERSION=3 fi export DIB_PYTHON_VERSION export DIB_PYTHON=python${DIB_PYTHON_VERSION} # Local variables: # mode: sh # End: diskimage-builder-2.11.0/diskimage_builder/elements/dib-python/pre-install.d/000077500000000000000000000000001324273057700271765ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dib-python/pre-install.d/01-dib-python000077500000000000000000000004611324273057700314200ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail python_path=$(command -v python${DIB_PYTHON_VERSION}) if [ -z "$python_path" ]; then echo "Could not find python${DIB_PYTHON_VERSION} executable." exit 1 fi ln -sf $python_path /usr/local/bin/dib-python diskimage-builder-2.11.0/diskimage_builder/elements/dib-run-parts/000077500000000000000000000000001324273057700251345ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dib-run-parts/README.rst000066400000000000000000000014251324273057700266250ustar00rootroot00000000000000============= dib-run-parts ============= .. warning:: This element is deprecated and is left only for compatibility. Please read the notes. This element install the ``dib-utils`` package to provide ``dib-run-parts``. Previously this element was a part of most base images and copied the internal version of ``dib-run-parts`` to ``/usr/local/bin`` during the build. Due to a (longstanding) oversight this was never removed and stayed in the final image. The image build process now uses a private copy of ``dib-run-parts`` during the build, so this element has become deprecated. For compatibility this element simply installs the ``dib-utils`` package, which will provide ``dib-run-parts``. However, this is probably better expressed as a dependency in individual elements. diskimage-builder-2.11.0/diskimage_builder/elements/dib-run-parts/package-installs.yaml000066400000000000000000000000121324273057700312330ustar00rootroot00000000000000dib-utils:diskimage-builder-2.11.0/diskimage_builder/elements/disable-selinux/000077500000000000000000000000001324273057700255355ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/disable-selinux/post-install.d/000077500000000000000000000000001324273057700304105ustar00rootroot0000000000000015-disable-selinux000077500000000000000000000003031324273057700335660ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/disable-selinux/post-install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail config=/etc/selinux/config [ -e $config ] && sed -i "s%^\(SELINUX=\s*\).*$%SELINUX=disabled%" $config diskimage-builder-2.11.0/diskimage_builder/elements/dkms/000077500000000000000000000000001324273057700234035ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dkms/README.rst000066400000000000000000000004601324273057700250720ustar00rootroot00000000000000==== dkms ==== This is the dkms (Dynamic Kernel Module System) element. Some distributions such as Fedora and Ubuntu include DKMS in their packaging. In these distros, it is reasonable to include dkms. Other RHEL based derivatives do not include DKMS, so those distros should not use the DKMS element. diskimage-builder-2.11.0/diskimage_builder/elements/dkms/element-deps000066400000000000000000000000211324273057700257010ustar00rootroot00000000000000package-installs diskimage-builder-2.11.0/diskimage_builder/elements/dkms/package-installs.yaml000066400000000000000000000000061324273057700275050ustar00rootroot00000000000000dkms: diskimage-builder-2.11.0/diskimage_builder/elements/dkms/post-install.d/000077500000000000000000000000001324273057700262565ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dkms/post-install.d/97-dkms000077500000000000000000000015001324273057700273730ustar00rootroot00000000000000#!/bin/bash # Trigger a run of dkms for all the modules installed # to ensure we have valid modules build for all. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail modules=$(dkms status | tr ',:' ' ' | awk '{ print $1 "/" $2 }') kernels=$(ls /usr/src/linux-headers-*-*-* -d | sed -e 's|/usr/src/linux-headers-||' || echo "") # NOTE(bnemec): On Fedora, the versions can be found in /usr/src/kernels if [ -z "$kernels" ]; then kernels=$(ls /usr/src/kernels/* -d | sed -e 's|/usr/src/kernels/||' || echo "") fi if [ -z "$kernels" ]; then echo "Warning: No kernel versions found for DKMS" fi __ARCH=$ARCH unset ARCH for module in $modules ; do for kernel in $kernels ; do dkms build $module -k $kernel dkms install $module -k $kernel done done ARCH=$__ARCH dkms status diskimage-builder-2.11.0/diskimage_builder/elements/docker/000077500000000000000000000000001324273057700237145ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/docker/README.rst000066400000000000000000000012121324273057700253770ustar00rootroot00000000000000====== docker ====== Base element for creating images from docker containers. This element is incomplete by itself, you'll want to add additional elements, such as dpkg or yum to get richer features. At its heart, this element simply exports a root tarball from a named docker container so that other diskimage-builder elements can build on top of it. The variables `DISTRO_NAME` and `DIB_RELEASE` will be used to decide which docker image to pull, and are required for most other elements. Additionally, the `DIB_DOCKER_IMAGE` environment variable can be set in addition to `DISTRO_NAME` and `DIB_RELEASE` if a different docker image is desired. diskimage-builder-2.11.0/diskimage_builder/elements/docker/element-provides000066400000000000000000000000211324273057700271120ustar00rootroot00000000000000operating-system diskimage-builder-2.11.0/diskimage_builder/elements/docker/extra-data.d/000077500000000000000000000000001324273057700261705ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/docker/extra-data.d/01-docker-minimal000077500000000000000000000022631324273057700312320ustar00rootroot00000000000000#!/bin/bash # # Copyright 2015 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail # This is not using install-packages because it's before install-packages can # work - and is work needed to make it possible for it to work if [ -x $TMP_MOUNT_PATH/usr/bin/apt-get ] ; then sudo chroot $TMP_MOUNT_PATH apt-get update sudo chroot $TMP_MOUNT_PATH apt-get -y install python elif [ -x $TMP_MOUNT_PATH/bin/dnf ] ; then sudo chroot $TMP_MOUNT_PATH dnf install -y findutils elif [ -x $TMP_MOUNT_PATH/usr/bin/yum ] ; then sudo chroot $TMP_MOUNT_PATH yum install -y findutils fi diskimage-builder-2.11.0/diskimage_builder/elements/docker/root.d/000077500000000000000000000000001324273057700251215ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/docker/root.d/08-docker000077500000000000000000000021071324273057700265430ustar00rootroot00000000000000#!/bin/bash # # Copyright 2015 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail if [ -f ${TARGET_ROOT}/.extra_settings ] ; then . ${TARGET_ROOT}/.extra_settings fi [ -n "$DISTRO_NAME" ] [ -n "$DIB_RELEASE" ] DIB_DOCKER_IMAGE=${DIB_DOCKER_IMAGE:-${DISTRO_NAME}:${DIB_RELEASE}} container=$(docker run -d $DIB_DOCKER_IMAGE /bin/sh) docker export $container | sudo tar -C $TARGET_ROOT --numeric-owner -xf - docker rm $container sudo rm -f ${TARGET_ROOT}/.extra_settings diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/000077500000000000000000000000001324273057700233725ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/README.rst000066400000000000000000000027621324273057700250700ustar00rootroot00000000000000==== dpkg ==== Provide dpkg specific image building glue. The ubuntu element needs customisations at the start and end of the image build process that do not apply to RPM distributions, such as using the host machine HTTP proxy when installing packages. These customisations live here, where they can be used by any dpkg based element. The dpkg specific version of install-packages is also kept here. Environment Variables --------------------- DIB_ADD_APT_KEYS :Required: No :Default: None :Description: If an extra or updated apt key is needed then define ``DIB_ADD_APT_KEYS`` with the path to a folder. Any key files inside will be added to the key ring before any apt-get commands take place. :Example: ``DIB_ADD_APT_KEYS=/etc/apt/trusted.gpg.d`` DIB_APT_LOCAL_CACHE :Required: No :Default: 1 :Description: By default the ``$DIB_IMAGE_CACHE/apt/$DISTRO_NAME`` directory is mounted in ``/var/cache/apt/archives`` to cache the .deb files downloaded during the image creation. Use this variable if you wish to disable the internal cache of the ``/var/cache/apt/archives`` directory :Example: ``DIB_APT_LOCAL_CACHE=0`` will disable internal caching. DIB_DISABLE_APT_CLEANUP :Required: No :Default: 0 :Description: At the end of a dib run we clean the apt cache to keep the image size as small as possible. Use this variable to prevent cleaning the apt cache at the end of a dib run. :Example: ``DIB_DISABLE_APT_CLEANUP=1`` will disable cleanup. .. element_deps:: diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/bin/000077500000000000000000000000001324273057700241425ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/bin/install-packages000077500000000000000000000041101324273057700273060ustar00rootroot00000000000000#!/bin/bash # Copyright 2012 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # install-packages package [package ...] ACTION=install MAP_ELEMENT="" SCRIPTNAME=$(basename $0) function show_options () { echo "Usage: $SCRIPTNAME [package ...]" echo echo "Options:" echo " -u -- update all packages" echo " -e -- erase/remove packages" echo " -m -- use custom element package map (Example: -m nova)" exit 0 } install_deb_packages () { DEBIAN_FRONTEND=noninteractive \ http_proxy=${http_proxy:-} https_proxy=${https_proxy:-} \ no_proxy=${no_proxy:-} \ apt-get --option "Dpkg::Options::=--force-confold" \ --option "Dpkg::Options::=--force-confdef" --assume-yes "$@" } TEMP=$(getopt -o hudem: -n $SCRIPTNAME -- "$@") if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi eval set -- "$TEMP" while true ; do case "$1" in -u) install_deb_packages dist-upgrade; exit 0;; -e) ACTION="remove"; shift;; -m) MAP_ELEMENT=$2; shift 2;; -h) show_options;; --) shift; break ;; *) echo "Error: unsupported option $1."; exit 1;; esac done PKGS=$@ if [ -n "$MAP_ELEMENT" ]; then if ! PKGS=$(pkg-map --element $MAP_ELEMENT $@); then echo "bin/pkg-map error. $PKGS" exit 1 fi fi if [ -z "${PKGS}" ]; then echo "Not running install-packages $ACTION with empty packages list" else install_deb_packages $ACTION $PKGS fi diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/cleanup.d/000077500000000000000000000000001324273057700252435ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/cleanup.d/40-unblock-daemons000077500000000000000000000005541324273057700304770ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail [ -n "$TARGET_ROOT" ] sudo mv $TARGET_ROOT/sbin/start-stop-daemon.REAL $TARGET_ROOT/sbin/start-stop-daemon if [ -f $TARGET_ROOT/sbin/initctl.REAL ]; then sudo mv $TARGET_ROOT/sbin/initctl.REAL $TARGET_ROOT/sbin/initctl fi sudo rm $TARGET_ROOT/usr/sbin/policy-rc.d diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/cleanup.d/50-remove-img-build-proxy000077500000000000000000000003121324273057700317320ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail [ -n "$TARGET_ROOT" ] # Undo our proxy support sudo rm -f $TARGET_ROOT/etc/apt/apt.conf.d/60img-build-proxy diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/cleanup.d/60-untrim-dpkg000077500000000000000000000003401324273057700276520ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail [ -n "$TARGET_ROOT" ] sudo rm $TARGET_ROOT/etc/dpkg/dpkg.cfg.d/02apt-speedup sudo rm $TARGET_ROOT/etc/apt/apt.conf.d/no-languages diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/element-deps000066400000000000000000000000471324273057700257000ustar00rootroot00000000000000install-bin manifests package-installs diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/extra-data.d/000077500000000000000000000000001324273057700256465ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/extra-data.d/01-copy-apt-keys000077500000000000000000000021451324273057700305210ustar00rootroot00000000000000#!/bin/bash # # Copyright 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail DIB_ADD_APT_KEYS=${DIB_ADD_APT_KEYS:-""} if [ -z "${DIB_ADD_APT_KEYS}" ]; then echo "DIB_ADD_APT_KEYS is not set - not importing keys" exit 0 fi DIR=${TMP_MOUNT_PATH}/tmp/apt_keys if [ -e ${DIR} ]; then echo "${DIR} already exists!" exit 1 fi sudo mkdir -p ${DIR} # dib-lint: safe_sudo # Copy to DIR for KEY in $(find ${DIB_ADD_APT_KEYS} -type f); do sudo cp -L ${KEY} ${DIR} # dib-lint: safe_sudo done diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/finalise.d/000077500000000000000000000000001324273057700254065ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/finalise.d/99-clean-up-cache000077500000000000000000000003301324273057700303340ustar00rootroot00000000000000#!/bin/bash # Do an apt-get clean. This will free some space. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail if [ "${DIB_DISABLE_APT_CLEANUP:-0}" != "1" ]; then apt-get clean fi diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/finalise.d/99-write-dpkg-manifest000077500000000000000000000025301324273057700314540ustar00rootroot00000000000000#!/bin/bash # # Copyright 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail DPKG_MANIFEST_NAME=dib-manifest-dpkg-$(basename ${IMAGE_NAME}) # Write the json preamble echo '{"packages": [' > ${DIB_MANIFEST_IMAGE_DIR}/${DPKG_MANIFEST_NAME} # Write a json-format output for the packages installed format='{"package": "${binary:Package}","arch": "${Architecture}","version": "${Version}","status": "${db:Status-Abbrev}"},' # Use the configurable dpkg --show output as a manifest dpkg-query --show --showformat="${format}" | tee -a ${DIB_MANIFEST_IMAGE_DIR}/${DPKG_MANIFEST_NAME} # Finish off the json structure and make it easier for human inspection sed -i -e 's/,$/\n]}/g' -e 's/\(},\)/ \1\n/g' ${DIB_MANIFEST_IMAGE_DIR}/${DPKG_MANIFEST_NAME} diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/pre-install.d/000077500000000000000000000000001324273057700260465ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/pre-install.d/00-disable-apt-recommends000077500000000000000000000004101324273057700325230ustar00rootroot00000000000000#!/bin/bash # Prevent apt from installing recommended packages if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail dd of=/etc/apt/apt.conf.d/95disable-recommends << _EOF_ APT::Install-Recommends "0"; Apt::Install-Suggests "0"; _EOF_ diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/pre-install.d/02-add-apt-keys000077500000000000000000000017371324273057700305060ustar00rootroot00000000000000#!/bin/bash # # Copyright 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail KEY_DIRECTORY=/tmp/apt_keys if [ ! -d "${KEY_DIRECTORY}" ]; then exit 0 fi for KEY in ${KEY_DIRECTORY}/*; do if ! file -b "${KEY}" | grep -q 'PGP public key block'; then echo "Skipping ${KEY}, not a valid GPG public key" continue fi apt-key add ${KEY} done diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/pre-install.d/99-apt-get-update000077500000000000000000000002521324273057700310530ustar00rootroot00000000000000#!/bin/bash # Do an apt-get update, so that packages can be installed. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail apt-get -y update diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/root.d/000077500000000000000000000000001324273057700245775ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dpkg/root.d/50-block-apt-translations000077500000000000000000000007461324273057700313510ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail [ -n "$TARGET_ROOT" ] # Configure APT not to fetch translations files sudo dd of=$TARGET_ROOT/etc/apt/apt.conf.d/95no-translations < /dev/null # and remove the translations, too echo 'Acquire::Languages "none";' | sudo tee $TARGET_ROOT/etc/apt/apt.conf.d/no-languages > /dev/null diskimage-builder-2.11.0/diskimage_builder/elements/dracut-network/000077500000000000000000000000001324273057700254165ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dracut-network/README.rst000066400000000000000000000002271324273057700271060ustar00rootroot00000000000000============== dracut-network ============== This element was removed in the Pike cycle. Please consider using the dracut-regenerate element instead. diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/000077500000000000000000000000001324273057700253575ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/README.rst000066400000000000000000000014411324273057700270460ustar00rootroot00000000000000============== dracut-ramdisk ============== Build Dracut-based ramdisks This is an alternative to the `ramdisk` element that uses Dracut to provide the base system functionality instead of Busybox. For elements that need additional drivers in the ramdisk image, a dracut-drivers.d feature is included that works in a similar fashion to the binary-deps.d feature. The element needing to add drivers should create a dracut-drivers.d directory and populate it with a single file listing all of the kernel modules it needs added to the ramdisk. Comments are not supported in this file. Note that these modules must be installed in the chroot first. By default, the virtio, virtio_net, and virtio_blk modules are included so that ramdisks are able to function properly in a virtualized environment. diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/binary-deps.d/000077500000000000000000000000001324273057700300165ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/binary-deps.d/dracut-ramdisk000066400000000000000000000000751324273057700326550ustar00rootroot00000000000000tail head awk ifconfig cut expr route ping nc wget tftp grep diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/dracut-drivers.d/000077500000000000000000000000001324273057700305375ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/dracut-drivers.d/base-drivers000066400000000000000000000000351324273057700330460ustar00rootroot00000000000000virtio virtio_net virtio_blk diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/element-deps000066400000000000000000000000721324273057700276630ustar00rootroot00000000000000deploy-targetcli pkg-map ramdisk-base source-repositories diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/environment.d/000077500000000000000000000000001324273057700301455ustar00rootroot0000000000000010-dracut-version.bash000066400000000000000000000001261324273057700341070ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/environment.d# Using 037 because that matches the current Fedora package export DRACUT_VERSION=037 diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/extra-data.d/000077500000000000000000000000001324273057700276335ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/extra-data.d/scripts/000077500000000000000000000000001324273057700313225ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/extra-data.d/scripts/module/000077500000000000000000000000001324273057700326075ustar00rootroot00000000000000deploy-cmdline.sh000077500000000000000000000010371324273057700357750ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/extra-data.d/scripts/module#!/bin/bash # NOTE(bnemec): Dracut doesn't like it if we enable these # dib-lint: disable=setu sete setpipefail dibdebugtrace # We never let Dracut boot off the specified root anyway, so all # we need is a value it will accept. root=/dev/zero rootok=1 # Dracut doesn't correctly parse the ip argument passed to us. # Override /proc/cmdline to rewrite it in a way dracut can grok. sed 's/\(ip=\S\+\)/\1:::off/' /proc/cmdline > /run/cmdline mount -n --bind -o ro /run/cmdline /proc/cmdline # Force Dracut to re-read the cmdline args CMDLINE= module-setup.sh000077500000000000000000000004611324273057700355130ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/extra-data.d/scripts/module#!/bin/bash # Dracut is bash too, and it doesn't play nicely with our usual sets # dib-lint: disable=setu sete setpipefail dibdebugtrace check() { return 0 } depends() { return 0 } install() { inst_hook cmdline 80 "$moddir/deploy-cmdline.sh" inst_hook pre-mount 50 "$moddir/init.sh" } diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/init.d/000077500000000000000000000000001324273057700265445ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/init.d/00-override-troubleshoot000066400000000000000000000003701324273057700332520ustar00rootroot00000000000000# NOTE(bnemec): Dracut provides its own emergency shell, and it doesn't # seem to allow directly starting bash as the existing troubleshoot function # does. function troubleshoot() { echo "Starting troubleshooting shell." emergency_shell } diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/install.d/000077500000000000000000000000001324273057700272475ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/install.d/20-install-dracut-deps000077500000000000000000000012351324273057700332740ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail DRACUT_NETWORK="dracut-network" if [ 'ubuntu' = "$DISTRO_NAME" ]; then DRACUT_NETWORK= # Install Dracut from source because the packaged version is # broken and old. For Dracut builds we throw away the chroot # anyway so it won't matter if we've installed some build deps. install-packages build-essential arping pushd /tmp tar xJvf dracut.tar.xz pushd dracut-$DRACUT_VERSION ./configure make make install popd popd fi PACKAGES=$(pkg-map --element dracut-ramdisk ncat tftp) install-packages $DRACUT_NETWORK wget $PACKAGES diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/pkg-map000066400000000000000000000003721324273057700266400ustar00rootroot00000000000000{ "family":{ "debian":{ "ncat":"netcat-traditional", "tftp":"tftp-hpa" }, "redhat":{ "ncat":"nmap-ncat" }, "suse":{ "arping": "iputils" } }, "default":{ "ncat":"ncat", "tftp":"tftp" } } diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/post-install.d/000077500000000000000000000000001324273057700302325ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/post-install.d/01-ensure-drivers000077500000000000000000000011611324273057700333520ustar00rootroot00000000000000#!/bin/bash set -eu set -o pipefail if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi TARGET_DIR="/tmp/in_target.d/" EXTRA_DRIVERS= for _FILE in $(ls ${TARGET_DIR}/dracut-drivers.d/) ; do _FILE="${TARGET_DIR}/dracut-drivers.d/${_FILE}" if [ -s $_FILE ]; then for _LINE in $(cat $_FILE) ; do EXTRA_DRIVERS="${EXTRA_DRIVERS} $_LINE" done fi done if [ "$EXTRA_DRIVERS" = "" ]; then echo "No extra drivers found" else DRIVERS_OUTPUT="/etc/dib_dracut_drivers" echo "Creating extra drivers record at: ${DRIVERS_OUTPUT}" echo "$EXTRA_DRIVERS" >${DRIVERS_OUTPUT} fi 99-build-dracut-ramdisk000077500000000000000000000056761324273057700343650ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/post-install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail _LIB="/tmp/ramdisk-build" source "$_LIB/common-defaults" source "$_LIB/img-defaults" source "$_LIB/ramdisk-defaults" source "$_LIB/common-functions" source "$_LIB/img-functions" source "$_LIB/ramdisk-functions" KERNEL_VERSION="${DIB_KERNEL_VERSION:-$(find_kernel_version)}" MODULE_DIR="$MODULE_ROOT/lib/modules/$KERNEL_VERSION" FIRMWARE_DIR="$MODULE_ROOT/lib/firmware" LIB_UDEV="$LIB_UDEV_ROOT/lib/udev" INIT="$_LIB/scripts/init" FUNCTIONS_D="$_LIB/scripts/d" MODULE="$_LIB/scripts/module" # NOTE(bnemec): IMAGE_ELEMENT is normally set in disk-image-create, but we're # not using that to build the image here. IMAGE_ELEMENT= mk_build_dir mkdir -p "$TMP_BUILD_DIR/mnt" export TMP_HOOKS_PATH=/tmp export TMP_MOUNT_PATH="$TMP_BUILD_DIR/mnt" echo "building ramdisk in $TMP_MOUNT_PATH" populate_init SCRIPT_HOME=/tmp/in_target.d/bin TMP_HOOKS_PATH=/tmp/in_target.d run_d ramdisk-install MODULE_PATH="/usr/lib/dracut/modules.d" cp -r "$MODULE" "$MODULE_PATH/80deploy-ramdisk" mv "$TMP_MOUNT_PATH/init" "$MODULE_PATH/80deploy-ramdisk/init.sh" # NOTE(bnemec): Workaround for bug in Dracut. This fix has been proposed to # Dracut as well in https://github.com/haraldh/dracut/pull/34 , and once it has # been released for all of our supported platforms we can remove this. Until then # this makes --include work correctly and will be a noop if we're running a fixed # Dracut version. sed -i 's|cp --reflink=auto --sparse=auto -fa -t "$s" "$i"$|cp --reflink=auto --sparse=auto -fa -t "${initdir}/${tgt}" "$i"|g' $(type -p dracut) # Notes on the options passed to Dracut: # -N: Do not build a host-specific ramdisk. We want to be able to run this ramdisk # on as many different machines as possible. # --install: A list of the binaries needed by our ramdisk script fragments. # --kernel-cmdline: Kernel parameters to pass to the ramdisk. rd.neednet is required # to force Dracut to bring up networking even if it isn't passed a # network target root. Pre-loading the ahci module is necessary # on some systems to ensure that SATA disks are recognized. # --include: Files from the chroot to include in the ramdisk. # --kver: The kernel version to use, as determined above. # --add-drivers: Extra kernel modules to include in the ramdisk. # -o: Force omission of these dracut modules. Our scripts are written for bash, # so dash is not desirable, and plymouth was causing some issues on Ubuntu. dracut -N \ --install "$(cat /etc/dib_binary_deps)" \ --kernel-cmdline "rd.shell rd.debug rd.neednet=1 rd.driver.pre=ahci" \ --include "$TMP_MOUNT_PATH/" / \ --kver "${KERNEL_VERSION}" \ --add-drivers "$(cat /etc/dib_dracut_drivers)" \ -o "dash plymouth" \ /tmp/ramdisk cp "/boot/vmlinuz-${KERNEL_VERSION}" /tmp/kernel chmod o+r /tmp/kernel # We don't want to run cleanup (set in mk_build_dir above) when this exits trap EXIT diskimage-builder-2.11.0/diskimage_builder/elements/dracut-ramdisk/source-repository-dracut000066400000000000000000000001601324273057700322740ustar00rootroot00000000000000dracut file /tmp/dracut.tar.xz https://www.kernel.org/pub/linux/utils/boot/dracut/dracut-$DRACUT_VERSION.tar.xz diskimage-builder-2.11.0/diskimage_builder/elements/dracut-regenerate/000077500000000000000000000000001324273057700260465ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dracut-regenerate/README.rst000066400000000000000000000007441324273057700275420ustar00rootroot00000000000000================= dracut-regenerate ================= Adds the possibility of regenerating dracut on image build time, giving the possibility to load extra modules. It relies on the ``DIB_DRACUT_ENABLED_MODULES`` setting, that will accept a yaml blob with the following format:: - name: packages: - - - name: packages: - - By default, this element will bring lvm and crypt modules. diskimage-builder-2.11.0/diskimage_builder/elements/dracut-regenerate/element-deps000066400000000000000000000000541324273057700303520ustar00rootroot00000000000000package-installs select-boot-kernel-initrd diskimage-builder-2.11.0/diskimage_builder/elements/dracut-regenerate/environment.d/000077500000000000000000000000001324273057700306345ustar00rootroot0000000000000010-dracut-regenerate000066400000000000000000000004151324273057700343170ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dracut-regenerate/environment.dexport DIB_DRACUT_ENABLED_MODULES_DEFAULT_CONFIG=" - name: crypt packages: - cryptsetup - name: lvm packages: - lvm2 " DIB_DRACUT_ENABLED_MODULES=${DIB_DRACUT_ENABLED_MODULES:-${DIB_DRACUT_ENABLED_MODULES_DEFAULT_CONFIG}} export DIB_DRACUT_ENABLED_MODULES diskimage-builder-2.11.0/diskimage_builder/elements/dracut-regenerate/finalise.d/000077500000000000000000000000001324273057700300625ustar00rootroot0000000000000050-dracut-regenerate000077500000000000000000000045141324273057700335600ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dracut-regenerate/finalise.d#!/usr/local/bin/dib-python # Copyright 2017 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import os import re import subprocess import yaml def main(): dracut_env = os.getenv('DIB_DRACUT_ENABLED_MODULES') dracut_objects = yaml.safe_load(dracut_env) modules_to_boot = [] for dracut_object in dracut_objects: # first, install dependent packages packages = dracut_object.get('packages', []) for package in packages: cmdline = ["install-packages", package] subp = subprocess.Popen(cmdline, stdout=subprocess.PIPE) out = subp.communicate()[0] if subp.returncode: e = subprocess.CalledProcessError(subp.returncode, cmdline) e.output = out raise e # second, compose the list of modules to boot modules_to_boot.append(dracut_object.get('name', None)) # regenerate dracut with the list of installed modules if len(modules_to_boot) > 0: cmdline = ["select-boot-kernel-initrd"] subp = subprocess.Popen(cmdline, stdout=subprocess.PIPE) out, err = subp.communicate() if subp.returncode: e = subprocess.CalledProcessError(subp.returncode, cmdline) e.output = out raise e kernel_set = out.decode().split(':') kernel_search = re.match("vmlinuz-(.*)", kernel_set[0]) kernel_version = "%s" % kernel_search.groups(1) ramdisk_path = "/boot/%s" % kernel_set[1].strip() modules_to_boot = ' ' .join(modules_to_boot) subp = subprocess.Popen(['dracut', '--force', '--add', modules_to_boot, '-f', ramdisk_path, kernel_version], stdout=subprocess.PIPE) subp.wait() if __name__ == '__main__': main() diskimage-builder-2.11.0/diskimage_builder/elements/dracut-regenerate/package-installs.yaml000066400000000000000000000002161324273057700321530ustar00rootroot00000000000000PyYAML: phase: pre-install.d dib_python_version: 2 python3-PyYAML: phase: pre-install.d installtype: package dib_python_version: 3 diskimage-builder-2.11.0/diskimage_builder/elements/dracut-regenerate/pkg-map000066400000000000000000000006451324273057700273320ustar00rootroot00000000000000{ "family": { "gentoo": { "PyYAML": "dev-python/pyyaml" }, "suse": { "PyYAML": "python-PyYAML", "python3-PyYAML": "python3-PyYAML" }, "redhat": { "PyYAML": "PyYAML", "python3-PyYAML": "python3-PyYAML" }, "debian": { "PyYAML": "python-yaml", "python3-PyYAML": "python3-yaml" } }, "default": { "PyYAML": "", "python3-PyYAML": "" } } diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/000077500000000000000000000000001324273057700251775ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/README.rst000066400000000000000000000036531324273057700266750ustar00rootroot00000000000000============= dynamic-login ============= This element insert a helper script in the image that allows users to dynamically configure credentials at boot time. This is specially useful for troubleshooting. Troubleshooting an image can be quite hard, specially if you can not get a prompt you can enter commands to find out what went wrong. By default, the images (specially ramdisks) doesn't have any SSH key or password for any user. Of course one could use the ``devuser`` element to generate an image with SSH keys and user/password in the image but that would be a massive security hole and very it's discouraged to run in production with a ramdisk like that. This element allows the operator to inject a SSH key and/or change the root password dynamically when the image boots. Two kernel command line parameters are used to do it: sshkey :Description: If the operator append sshkey="$PUBLIC_SSH_KEY" to the kernel command line on boot, the helper script will append this key to the root user authorized_keys. rootpwd :Description: If the operator append rootpwd="$ENCRYPTED_PASSWORD" to the kernel command line on boot, the helper script will set the root password to the one specified by this option. Note that this password must be **encrypted**. Encrypted passwords can be generated using the ``openssl`` command, e.g: *openssl passwd -1*. .. note:: The value of these parameters must be **quoted**, e.g: sshkey="ssh-rsa BBBA1NBzaC1yc2E ..." .. warning:: Some base operational systems might require selinux to be in **permissive** or **disabled** mode so that you can log in the image. This can be achieved by building the image with the ``selinux-permissive`` element for diskimage-builder or by passing ``selinux=0`` in the kernel command line. RHEL/CentOS are examples of OSs which this is true. diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/element-deps000066400000000000000000000000561324273057700275050ustar00rootroot00000000000000dib-init-system install-static openssh-server diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/init-scripts/000077500000000000000000000000001324273057700276275ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/init-scripts/systemd/000077500000000000000000000000001324273057700313175ustar00rootroot00000000000000dynamic-login.service000066400000000000000000000002341324273057700353530ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/init-scripts/systemd[Unit] Description=Dynamic Login After=network.target [Service] Type=oneshot ExecStart=/usr/local/bin/dynamic-login [Install] WantedBy=multi-user.target diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/init-scripts/sysv/000077500000000000000000000000001324273057700306335ustar00rootroot00000000000000dynamic-login.init000077500000000000000000000011101324273057700341670ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/init-scripts/sysv#!/bin/sh -e ### BEGIN INIT INFO # Provides: dynamic-login # Required-Start: $local_fs networking # Required-Stop: $local_fs # Default-Start: S # Default-Stop: 0 6 # X-Start-Before: # Short-Description: Dynamic Login # Description: Execute Dynamic Login ### END INIT INFO NAME=dynamic-login INIT_NAME=/etc/init.d/${NAME} SCRIPT_NAME=/usr/local/bin/${NAME} [ -x $SCRIPT_NAME ] || exit 0 case "$1" in start) $SCRIPT_NAME ;; stop) ;; *) echo "Usage: $INIT_NAME {start}" exit 1 ;; esac exit 0 diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/init-scripts/upstart/000077500000000000000000000000001324273057700313315ustar00rootroot00000000000000dynamic-login.conf000066400000000000000000000003021324273057700346460ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/init-scripts/upstartdescription "Dynamic Login" start on runlevel [2345] stop on runlevel [!2345] umask 022 expect stop script echo "Executing Dynamic Login" /usr/local/bin/dynamic-login end script diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/install.d/000077500000000000000000000000001324273057700270675ustar00rootroot0000000000000070-enable-dynamic-login-services000077500000000000000000000004131324273057700347570ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/install.d#!/bin/bash if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail case "$DIB_INIT_SYSTEM" in systemd) systemctl enable dynamic-login.service ;; sysv) update-rc.d dynamic-login.init defaults ;; esac diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/static/000077500000000000000000000000001324273057700264665ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/static/usr/000077500000000000000000000000001324273057700272775ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/static/usr/local/000077500000000000000000000000001324273057700303715ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/static/usr/local/bin/000077500000000000000000000000001324273057700311415ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/dynamic-login/static/usr/local/bin/dynamic-login000077500000000000000000000022131324273057700336170ustar00rootroot00000000000000#!/bin/bash # dib-lint: disable=dibdebugtrace # Copyright 2015 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. set -eu set -o pipefail # Reads an encrypted root password from the kernel command line and set # it to the root user if [[ $( $SSHDIR/authorized_keys chmod 600 $SSHDIR/authorized_keys fi diskimage-builder-2.11.0/diskimage_builder/elements/element-manifest/000077500000000000000000000000001324273057700257025ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/element-manifest/README.rst000066400000000000000000000004171324273057700273730ustar00rootroot00000000000000================ element-manifest ================ Writes a manifest file that is the full list of elements that were used to build the image. The file path can be overridden by setting $DIB\_ELEMENT\_MANIFEST\_PATH, and defaults to /etc/dib-manifests/element-manifest. diskimage-builder-2.11.0/diskimage_builder/elements/element-manifest/element-deps000066400000000000000000000000121324273057700302000ustar00rootroot00000000000000manifests diskimage-builder-2.11.0/diskimage_builder/elements/element-manifest/extra-data.d/000077500000000000000000000000001324273057700301565ustar00rootroot0000000000000075-inject-element-manifest000077500000000000000000000005531324273057700347700ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/element-manifest/extra-data.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail DIB_ELEMENT_MANIFEST_PATH=${DIB_ELEMENT_MANIFEST_PATH:-"$DIB_MANIFEST_IMAGE_DIR/dib-element-manifest"} sudo mkdir -p $TMP_MOUNT_PATH/$(dirname $DIB_ELEMENT_MANIFEST_PATH) sudo /bin/bash -c "echo $IMAGE_ELEMENT | tr ' ' '\n' > $TMP_MOUNT_PATH/$DIB_ELEMENT_MANIFEST_PATH" diskimage-builder-2.11.0/diskimage_builder/elements/enable-serial-console/000077500000000000000000000000001324273057700266105ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/enable-serial-console/README.rst000066400000000000000000000006051324273057700303000ustar00rootroot00000000000000===================== enable-serial-console ===================== Start getty on active serial consoles. With ILO and other remote admin environments, having a serial console can be useful for debugging and troubleshooting. For upstart: If ttyS1 exists, getty will run on that, otherwise on ttyS0. For systemd: We dynamically start a getty on any active serial port via udev rules. diskimage-builder-2.11.0/diskimage_builder/elements/enable-serial-console/cleanup.d/000077500000000000000000000000001324273057700304615ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/enable-serial-console/cleanup.d/99-fix-grub000066400000000000000000000007211324273057700323660ustar00rootroot00000000000000set -euxo BOOTDIR=$TARGET_ROOT/boot # Splashimage in grub has a known issue with serial console # So grub config file needs to be fixed. if [ -f $BOOTDIR/grub/grub.conf ] ; then sed -i -e "/^splashimage/d;s/ rhgb\( \|$\)/\1/g;s/ quiet\( \|$\)/\1/g;/^serial/d;/^terminal/d;/^hiddenmenu/d" $BOOTDIR/grub/grub.conf sed -i "/^default/aserial --unit=0 --speed=9600 --word=8 --parity=no --stop=1\nterminal --timeout=5 serial console" $BOOTDIR/grub/grub.conf fi diskimage-builder-2.11.0/diskimage_builder/elements/enable-serial-console/install.d/000077500000000000000000000000001324273057700305005ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/enable-serial-console/install.d/20-stty000077500000000000000000000006361324273057700316550ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail SCRIPTDIR=$(dirname $0) if [ "$DIB_INIT_SYSTEM" == "upstart" ]; then install -m 0644 -o root -g root ${SCRIPTDIR}/ttySx.conf /etc/init/ttySx.conf elif [ "$DIB_INIT_SYSTEM" == "systemd" ]; then install -D -g root -o root -m 0644 ${SCRIPTDIR}/serial-console-udev.rules /etc/udev/rules.d/99-serial-console.rules fi serial-console-udev.rules000066400000000000000000000001551324273057700353560ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/enable-serial-console/install.dSUBSYSTEM=="tty", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}+="getty@$name.service", ATTRS{type}=="4" diskimage-builder-2.11.0/diskimage_builder/elements/enable-serial-console/install.d/ttySx.conf000066400000000000000000000007041324273057700325030ustar00rootroot00000000000000# This service maintains a getty on ttyS0/1 from the point the system is # started until it is shut down again. start on stopped rc or RUNLEVEL=[2345] stop on runlevel [!2345] respawn script console_port=0 echo "ttySx probing ttyS1" >/dev/ttyS1 2>/dev/null && console_port=1 echo "ttySx: console tty = /dev/ttyS$console_port" >&2 exec /sbin/agetty -m ttyS$console_port 115200,57600,38400,9600,4800 vt102 end script diskimage-builder-2.11.0/diskimage_builder/elements/epel/000077500000000000000000000000001324273057700233725ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/epel/README.rst000066400000000000000000000014071324273057700250630ustar00rootroot00000000000000==== epel ==== This element installs the Extra Packages for Enterprise Linux (EPEL) repository GPG key as well as configuration for yum. Note this element only works with platforms that have EPEL support such as CentOS and RHEL DIB_EPEL_MIRROR: :Required: No :Default: None :Description: To use a EPEL Yum mirror, set this variable to the mirror URL before running bin/disk-image-create. This URL should point to the directory containing the ``5/6/7`` directories. :Example: ``DIB\_EPEL\_MIRROR=http://dl.fedoraproject.org/pub/epel`` DIB_EPEL_DISABLED: :Required: No :Default: 0 :Description: To disable the EPEL repo (but leave it available if used with an explicit ``--enablerepo``) set this to 1 diskimage-builder-2.11.0/diskimage_builder/elements/epel/element-deps000066400000000000000000000000211324273057700256700ustar00rootroot00000000000000package-installs diskimage-builder-2.11.0/diskimage_builder/elements/epel/package-installs.yaml000066400000000000000000000000771324273057700275040ustar00rootroot00000000000000wget: phase: pre-install.d yum-utils: phase: pre-install.d diskimage-builder-2.11.0/diskimage_builder/elements/epel/pre-install.d/000077500000000000000000000000001324273057700260465ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/epel/pre-install.d/05-rpm-epel-release000077500000000000000000000023071324273057700313570ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail if [[ ${DISTRO_NAME} =~ "centos" ]]; then # Centos has "epel-release" in extras, which is default enabled. yum install -y epel-release else # For RHEL, we have to scrape the download page to find the latest # release and install that [ -n "$ARCH" ] if [ 'amd64' = "$ARCH" ] ; then ARCH="x86_64" fi BASE_URL=${DIB_EPEL_MIRROR:-https://dl.fedoraproject.org/pub/epel} case "$DISTRO_NAME" in rhel7) RELEASE=7 URL=$BASE_URL/$RELEASE/x86_64/Packages/e/ ;; *) echo "$DISTRO_NAME is not supported" exit 1 ;; esac PKG_NAME=$(wget -q $URL -O - |grep -oE "(href=\"epel-release-$RELEASE-[0-9,.].*)" | cut -d'"' -f2) rpm -q epel-release || yum install -y $URL/$PKG_NAME fi if [ ${DIB_EPEL_DISABLED:-0} -ne 0 ]; then yum-config-manager --disable epel fi DIB_EPEL_MIRROR=${DIB_EPEL_MIRROR:-} [ -n "$DIB_EPEL_MIRROR" ] || exit 0 # Set the EPEL mirror to use sed -e "s|^#baseurl=http://download.fedoraproject.org/pub/epel|baseurl=$DIB_EPEL_MIRROR|;/^mirrorlist=/d" -i /etc/yum.repos.d/epel.repo diskimage-builder-2.11.0/diskimage_builder/elements/fedora-minimal/000077500000000000000000000000001324273057700253315ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/fedora-minimal/README.rst000066400000000000000000000012601324273057700270170ustar00rootroot00000000000000============== fedora-minimal ============== Create a minimal image based on Fedora. Use of this element will require 'yum' and 'yum-utils' to be installed on Ubuntu and Debian. Nothing additional is needed on Fedora or CentOS. The element will need `python-lzma` everywhere. Due to a bug in the released version of urlgrabber, on many systems an installation of urlgrabber from git is required. The git repository can be found here: http://yum.baseurl.org/gitweb?p=urlgrabber.git;a=summary This element sets the ``DIB_RELEASE`` var to 'fedora'. The release of fedora to be installed can be controlled through the ``DIB_RELEASE`` variable, which defaults the latest supported release. diskimage-builder-2.11.0/diskimage_builder/elements/fedora-minimal/element-deps000066400000000000000000000000141324273057700276310ustar00rootroot00000000000000yum-minimal diskimage-builder-2.11.0/diskimage_builder/elements/fedora-minimal/element-provides000066400000000000000000000000211324273057700305270ustar00rootroot00000000000000operating-system diskimage-builder-2.11.0/diskimage_builder/elements/fedora-minimal/environment.d/000077500000000000000000000000001324273057700301175ustar00rootroot0000000000000010-fedora-distro-name.bash000066400000000000000000000001001324273057700345640ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/fedora-minimal/environment.dexport DISTRO_NAME=fedora export DIB_RELEASE=${DIB_RELEASE:-26} diskimage-builder-2.11.0/diskimage_builder/elements/fedora-minimal/environment.d/11-yum-dnf.bash000066400000000000000000000003161324273057700325540ustar00rootroot00000000000000# since f22, dnf is the yum replacement. Mostly it drops in # unmodified, so while we transition KISS and use this to choose if [ $DIB_RELEASE -ge 22 ]; then export YUM=dnf else export YUM=yum fi diskimage-builder-2.11.0/diskimage_builder/elements/fedora-minimal/test-elements/000077500000000000000000000000001324273057700301225ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/000077500000000000000000000000001324273057700330155ustar00rootroot00000000000000README.rst000066400000000000000000000000541324273057700344240ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/fedora-minimal/test-elements/build-succeedsVerify we can build a fedora-minimal image. element-deps000066400000000000000000000000241324273057700352370ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/fedora-minimal/test-elements/build-succeedsopenstack-ci-mirrorsdiskimage-builder-2.11.0/diskimage_builder/elements/fedora-minimal/yum.repos.d/000077500000000000000000000000001324273057700275145ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/fedora-minimal/yum.repos.d/yum.repo000066400000000000000000000011601324273057700312130ustar00rootroot00000000000000[fedora] name=Fedora $releasever - $basearch failovermethod=priority #baseurl=http://download.fedoraproject.org/pub/fedora/linux/releases/$releasever/Everything/$basearch/os/ metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch gpgcheck=0 skip_if_unavailable=False [updates] name=Fedora $releasever - $basearch - Updates failovermethod=priority #baseurl=http://download.fedoraproject.org/pub/fedora/linux/updates/$releasever/$basearch/ metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f$releasever&arch=$basearch enabled=1 gpgcheck=0 skip_if_unavailable=False diskimage-builder-2.11.0/diskimage_builder/elements/fedora/000077500000000000000000000000001324273057700237055ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/fedora/README.rst000066400000000000000000000011561324273057700253770ustar00rootroot00000000000000====== fedora ====== Use Fedora cloud images as the baseline for built disk images. For further details see the redhat-common README. Environment Variables --------------------- DIB_DISTRIBUTION_MIRROR: :Required: No :Default: None :Description: To use a Fedora Yum mirror, set this variable to the mirror URL before running bin/disk-image-create. This URL should point to the directory containing the ``releases/updates/development`` and ``extras`` directories. :Example: ``DIB\_DISTRIBUTION\_MIRROR=http://download.fedoraproject.org/pub/fedora/linux`` diskimage-builder-2.11.0/diskimage_builder/elements/fedora/element-deps000066400000000000000000000001001324273057700262010ustar00rootroot00000000000000cache-url dkms redhat-common rpm-distro source-repositories yum diskimage-builder-2.11.0/diskimage_builder/elements/fedora/element-provides000066400000000000000000000000211324273057700271030ustar00rootroot00000000000000operating-system diskimage-builder-2.11.0/diskimage_builder/elements/fedora/environment.d/000077500000000000000000000000001324273057700264735ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/fedora/environment.d/10-fedora-distro-name.bash000066400000000000000000000014211324273057700332260ustar00rootroot00000000000000export DISTRO_NAME=fedora export DIB_RELEASE=${DIB_RELEASE:-27} # Note the filename URL has a "sub-release" in it # http:// ... Fedora-Cloud-Base-25-1.3.x86_64.qcow2 # ^^^ # It's not exactly clear how this is generated, or how we could # determine this programatically. Other projects have more # complicated regex-based scripts to find this, which we can examine # if this becomes an issue ... see thread at [1] # # [1] https://lists.fedoraproject.org/archives/list/cloud@lists.fedoraproject.org/thread/2WFO2FKIGUQYRQXIR35UVJGRHF7LQENJ/ if [[ ${DIB_RELEASE} == '26' ]]; then export DIB_FEDORA_SUBRELEASE=1.5 elif [[ ${DIB_RELEASE} == '27' ]]; then export DIB_FEDORA_SUBRELEASE=1.6 else echo "Unsupported Fedora release" exit 1 fi diskimage-builder-2.11.0/diskimage_builder/elements/fedora/environment.d/11-yum-dnf.bash000066400000000000000000000003161324273057700311300ustar00rootroot00000000000000# since f22, dnf is the yum replacement. Mostly it drops in # unmodified, so while we transition KISS and use this to choose if [ $DIB_RELEASE -ge 22 ]; then export YUM=dnf else export YUM=yum fi diskimage-builder-2.11.0/diskimage_builder/elements/fedora/package-installs.yaml000066400000000000000000000026251324273057700300200ustar00rootroot00000000000000# On a fresh Fedora 18 install you might have to update audit in order to # fix a conflict with a file from the glibc package. # https://bugzilla.redhat.com/show_bug.cgi?id=894307 audit: # The version of openssl shipped in the fedora cloud image is no longer # compatible with new python environments installed by virtualenv, so we need # to update it first. # See https://bugs.launchpad.net/diskimage-builder/+bug/1254879 openssl: # FIXME: To avoid conflict between the pyOpenSSL installed via python-pip # and pyOpenSSL installed via yum, we are going to sort it out installing # it earlier at the beginning of the image building process. Python-pip # is installing pyOpenSSL as part of satisfying the requirements.txt # in python-glanceclient and afterwards yum tries to install it as a # dependency of the python-paste package needed for the heat element, # this seems to be conflicting and causing the image building process to # fail. The problem is happening on a Fedora 18 system. pyOpenSSL: # Workaround for: # https://bugzilla.redhat.com/show_bug.cgi?id=1066983 vim-minimal: # kernel modules to match the core kernel # Newer Fedora 21 splits these out into a separate package. # It contains iscsi_tcp.ko (for Ironic) among other things like network # driver modules, etc. kernel-modules: # This small package is required on Fedora 21 in order to build some # packages via source (MySQL driver). redhat-rpm-config: diskimage-builder-2.11.0/diskimage_builder/elements/fedora/pre-install.d/000077500000000000000000000000001324273057700263615ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/fedora/pre-install.d/01-set-fedora-mirror000077500000000000000000000007151324273057700320710ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-} [ -n "$DIB_DISTRIBUTION_MIRROR" ] || exit 0 for FILE in /etc/yum.repos.d/fedora.repo /etc/yum.repos.d/fedora-updates.repo /etc/yum.repos.d/fedora-updates-testing.repo; do sudo sed -e "s|^#baseurl=http://download.fedoraproject.org/pub/fedora/linux|baseurl=$DIB_DISTRIBUTION_MIRROR|;/^metalink/d" -i $FILE done diskimage-builder-2.11.0/diskimage_builder/elements/fedora/pre-install.d/02-set-machine-id000077500000000000000000000011501324273057700313120ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # The systemd .spec file does this in %post [1] and it turns out that # (in what is possibly a bug [2]) that kernel install requires # /etc/machine-id. This affects "fedora" (the image-based build) if # there is no systemd update to install ... with "fedora-minimal" we # are always installing into the chroot so, so %post always runs and # this is always run # # [1] https://src.fedoraproject.org/rpms/systemd/blob/master/f/systemd.spec # [2] https://bugzilla.redhat.com/show_bug.cgi?id=1486124 systemd-machine-id-setup diskimage-builder-2.11.0/diskimage_builder/elements/fedora/root.d/000077500000000000000000000000001324273057700251125ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/fedora/root.d/10-fedora-cloud-image000077500000000000000000000022671324273057700307110ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail [ -n "$ARCH" ] [ -n "$TARGET_ROOT" ] if [ 'amd64' = "$ARCH" ] ; then ARCH="x86_64" fi DIB_LOCAL_IMAGE=${DIB_LOCAL_IMAGE:-""} if [ -n "$DIB_LOCAL_IMAGE" ]; then IMAGE_LOCATION=$DIB_LOCAL_IMAGE # No need to copy a local image into the cache directory, so just specify # the cached path as the original path. CACHED_IMAGE=$IMAGE_LOCATION BASE_IMAGE_FILE=`basename $DIB_LOCAL_IMAGE` BASE_IMAGE_TAR=$BASE_IMAGE_FILE.tgz else # note default DIB_RELEASE set in environment setup # Not sure if ${ARCH} is defined for anything but x86_64 here DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-https://download.fedoraproject.org/pub/fedora/linux/releases/${DIB_RELEASE}/CloudImages/${ARCH}/images} BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-Fedora-Cloud-Base-$DIB_RELEASE-$DIB_FEDORA_SUBRELEASE.$ARCH.qcow2} BASE_IMAGE_TAR=Fedora-Cloud-Base-$DIB_RELEASE-$DIB_FEDORA_SUBRELEASE.$ARCH.tgz IMAGE_LOCATION=$DIB_CLOUD_IMAGES/$BASE_IMAGE_FILE CACHED_IMAGE=$DIB_IMAGE_CACHE/$BASE_IMAGE_FILE fi $TMP_HOOKS_PATH/bin/extract-image $BASE_IMAGE_FILE $BASE_IMAGE_TAR $IMAGE_LOCATION $CACHED_IMAGE diskimage-builder-2.11.0/diskimage_builder/elements/fedora/test-elements/000077500000000000000000000000001324273057700264765ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/fedora/test-elements/build-succeeds/000077500000000000000000000000001324273057700313715ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/fedora/test-elements/build-succeeds/element-deps000066400000000000000000000000321324273057700336710ustar00rootroot00000000000000base openstack-ci-mirrors diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/000077500000000000000000000000001324273057700237405ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/README.rst000066400000000000000000000040761324273057700254360ustar00rootroot00000000000000======== Gentoo ======== Use a Gentoo cloud image as the baseline for built disk images. The images are located in profile specific sub directories: http://distfiles.gentoo.org/releases/amd64/autobuilds/ As of this writing, only x86_64 images are available. Notes: * There are very frequently new automated builds that include changes that happen during the product maintenance. The download directories contain an unversioned name and a versioned name. The unversioned name will always point to the latest image, but will frequently change its content. The versioned one will never change content, but will frequently be deleted and replaced by a newer build with a higher version-release number. * In order to run the package-installs element you will need to make sure `dev-python/pyyaml` is installed on the host. * In order to run the vm element you will need to make sure `sys-block/parted` is installed on the host. * Other profiles can be used by exporting GENTOO_PROFILE with a valid profile. A list of valid profiles follows: default/linux/amd64/17.0 default/linux/amd64/17.0/no-multilib hardened/linux/amd64 hardened/linux/amd64/no-multilib * You can set the GENTOO_PORTAGE_CLEANUP environment variable to true (or anything other than False) to clean up portage from the system and get the image size smaller. * Gentoo supports many diferent versions of python, in order to select one you may use the `GENTOO_PYTHON_TARGETS` environment variable to select the versions of python you want on your image. The format of this variable is a string as follows `"python2_7 python3_5"`. * In addition you can select the primary python version you wish to use (that which will be called by running the `python` command. The `GENTOO_PYTHON_ACTIVE_VERSION` is used to set that mapping. The variable contents can be something like `python3.5`. * You can enable overlays using the `GENTOO_OVERLAYS` variable. In it you should put a space separated list of overlays. The overlays must be in the official overlay list and must be git based. diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/bin/000077500000000000000000000000001324273057700245105ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/bin/fix_shm000077500000000000000000000025771324273057700261060ustar00rootroot00000000000000#!/bin/bash # Copyright 2017 Matthew Thode # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [[ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]]; then set -x fi set -eu set -o pipefail # make /dev/shm dir if it doesn't exist # mount tmpfs and chown it # existing programs could be using /dev/shm # This means it cannot be moved or backed # up as a copy easily. The only remaining # option is to move the link if it exists # as a link. Existing programs will still # hold the file handle of the original # location open and new programs can use # the fixed /dev/shm. [[ -e /run/lock/shm_fixed ]] && exit 0 # shm has already been fixed if [[ ! -d /dev/shm ]]; then if [[ ! -e /dev/shm ]]; then if [[ -L /dev/shm ]]; then mv /dev/shm /dev/shm.orig fi mkdir /dev/shm fi fi mount -t tmpfs none /dev/shm chmod 1777 /dev/shm mkdir -p /run/lock touch /run/lock/shm_fixed diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/bin/install-packages000077500000000000000000000056311324273057700276650ustar00rootroot00000000000000#!/bin/bash # Copyright 2016 Matthew Thode # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [[ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]]; then set -x fi set -eu set -o pipefail ACTION='install' SCRIPTNAME="$(basename $0)" function show_options { echo "Usage: ${SCRIPTNAME} [package ...]" echo echo 'Options:' echo ' -u -- update all packages' echo ' -e -- erase/remove packages' exit 0 } function install_gentoo_packages { RUN_ONCE_SHM='1' if mountpoint -q /dev/shm; then emerge $@ elif [[ -k /dev/shm ]]; then emerge $@ else fix_shm emerge $@ unfix_shm fi } TEMP="$(getopt -o huem: -n ${SCRIPTNAME} -- ${@})" if [[ "${?}" != 0 ]]; then echo 'Terminating...' >&2 exit 1 fi eval set -- "${TEMP}" while true; do case "${1}" in -u ) if [[ ! -f /usr/portage/profiles ]]; then emerge-webrsync -q fi install_gentoo_packages -uDNv --with-bdeps=y --complete-graph=y --jobs=2 @world install_gentoo_packages --verbose=n --depclean install_gentoo_packages -v --usepkg=n @preserved-rebuild etc-update --automode -5 eselect news read new exit 0; ;; -e ) ACTION='remove' shift ;; -m ) MAP_ELEMENT="${2}" shift 2 ;; -h ) show_options ;; -- ) shift break ;; * ) echo "Error: unsupported option ${1}." exit 1 ;; esac done PKGS="${@}" if [[ -n "${MAP_ELEMENT:-}" ]]; then PKGS="$(pkg-map --element ${MAP_ELEMENT} ${@})" if [[ "${?}" != 0 ]]; then echo "bin/pkg-map error. ${PKGS}" exit 1 fi fi if [[ -z "${PKGS}" ]]; then echo "Not running install-packages ${ACTION} with empty packages list" else if [[ "${ACTION}" == 'install' ]]; then if [[ ! -f /usr/portage/profiles ]]; then emerge-webrsync -q fi install_gentoo_packages -q --jobs=2 --changed-use "${PKGS}" elif [[ "${ACTION}" == 'remove' ]]; then if [[ ! -f /usr/portage/profiles ]]; then emerge-webrsync -q fi install_gentoo_packages -C "${PKGS}" else echo 'something when really wrong, install action is not install or remove' fi fi diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/bin/unfix_shm000077500000000000000000000016641324273057700264450ustar00rootroot00000000000000#!/bin/bash # Copyright 2017 Matthew Thode # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [[ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]]; then set -x fi set -eu set -o pipefail # unmount tmpfs # care about anything still using it [[ ! -e /run/lock/shm_fixed ]] && exit 0 # shm hasn't been modified umount /dev/shm if fuser /dev/shm; then rmdir /dev/shm fi if [[ -e /dev/shm.orig ]]; then mv /dev/shm.orig /dev/shm fi rm /run/lock/shm_fixed diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/element-deps000066400000000000000000000000471324273057700262460ustar00rootroot00000000000000cache-url install-bin package-installs diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/element-provides000066400000000000000000000000211324273057700271360ustar00rootroot00000000000000operating-system diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/environment.d/000077500000000000000000000000001324273057700265265ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash000066400000000000000000000006051324273057700323720ustar00rootroot00000000000000export DIB_RELEASE=gentoo export DISTRO_NAME=gentoo export GENTOO_PROFILE=${GENTOO_PROFILE:-'default/linux/amd64/17.0'} export GENTOO_PORTAGE_CLEANUP=${GENTOO_PORTAGE_CLEANUP:-'False'} export GENTOO_PYTHON_TARGETS=${GENTOO_PYTHON_TARGETS:-'python2_7 python3_5'} export GENTOO_PYTHON_ACTIVE_VERSION=${GENTOO_PYTHON_ACTIVE_VERSION:-'python3.5'} export GENTOO_OVERLAYS=${GENTOO_OVERLAYS:-''} diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/extra-data.d/000077500000000000000000000000001324273057700262145ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/extra-data.d/gentoo-releng.gpg000066400000000000000000000122331324273057700314610ustar00rootroot00000000000000-----BEGIN PGP PUBLIC KEY BLOCK----- mQINBEqUWzgBEACXftaG+HVuSQBEqdpBIg2SOVgWW/KbCihO5wPOsdbM93e+psmb wvw+OtNHADQvxocKMuZX8Q/j5i3nQ/ikQFW5Oj6UXvl1qyxZhR2P7GZSNQxn0eMI zAX08o691ws2/dFGXKmNT6btYJ0FxuTtTVSK6zi68WF+ILGK/O2TZXK9EKfZKPDH KHcGrUq4c03vcGANz/8ksJj2ZYEGxMr1h7Wfe9PVcm0gCB1MhYHNR755M47V5Pch fyxbs6vaKz82PgrNjjjbT0PISvnKReUOdA2PFUWry6UKQkiVrLVDRkd8fryLL8ey 5JxgVoJZ4echoVWQ0JYJ5lJTWmcZyxQYSAbz2w9dLB+dPyyGpyPp1KX1ADukbROp 9S11I9+oVnyGdUBm+AUme0ecekWvt4qiCw3azghLSwEyGZc4a8nNcwSqFmH7Rqdd 1+gHc+4tu4WHmguhMviifGXKyWiRERULp0obV33JEo/c4uwyAZHBTJtKtVaLb92z aRdh1yox2I85iumyt62lq9dfOuet4NNVDnUkqMYCQD23JB8IM+qnVaDwJ6oCSIKi nY3uyoqbVE6Lkm+Hk5q5pbvg1cpEH6HWWAl20EMCzMOoMcH0tPyQLDlD2Mml7veG kwdy3S6RkjCympbNzqWec2+hkU2c93Bgpfh7QP0GDN0qrzcNNFmrD5795QARAQAB tFNHZW50b28gTGludXggUmVsZWFzZSBFbmdpbmVlcmluZyAoQXV0b21hdGVkIFdl ZWtseSBSZWxlYXNlIEtleSkgPHJlbGVuZ0BnZW50b28ub3JnPoheBBARCAAGBQJR yaWCAAoJELP3uHLZaXat08sA/3paxuDydIV/8qe9PzgID6zifip9T9XfTDCRbHQR Kw0xAP9vTE9yoPuNMrF55AP9+68FbYaIO0sUxNN9CVby28iU7IkBHAQQAQIABgUC Ut9qEwAKCRAvJnjSPF2apBtSCACIPmfvwiBluwx1dz4/C4pUSIZmaRk5NrKhuADL CBUyH4X/Ajz4MhvXjAYeWpvtzxHW5sJL0mnNBQtkEM3SPGrYsJLCmhp0hW0lfYtq pG8Kymej7N7CJMYKW65HTvlLyCM1JpBy3OAXBgtxNIeho+dXbXTBPAUCje2MVS8h tFgxn6mmXqQPh3YQTY8UE4c+s4XOLBiV2PQowmRZ/m7OzPTf7Yp9j/WJdJN6Rn8P lOvsQ6soThiFm5kr1UqreALKEZOVrWT7SuhNFB3s8luHfKkwiWx4B2/Pku9LOXEz Z6hnOV+ZWsF9LQEGtTmS3BZVIVEEKniBLEnoqPdae5xHhJhXiQIcBBABAgAGBQJO ucCvAAoJENpWdD1eHU4JCY0P/2PU2WDPI1I1/fO6O7jflQMXkxrKi7IP/VaELTlg ADhKRDecWOJltodAoIZItLowviRH0N5FQ7a9MtYG3DyDZPvkF7U0UXoej8uT0XBp FRLqbSbGL3krnXR0RfFsolK4x53nXGDV9noZsOWEovafhAysOngLCCNk136fSI3l gEQngJ6ChUN0IWtkQaE2IATOqgwkGQ7jJjt5qSznadCqBTaOkHvKCM4KZOjJC/zG /ZIQ2+PmFtR+LpgqdugHzGHvP+gG8jEG/i4EX8aAzcDVP8tejAqsS2X1yRSY7GEf odJ9679dUwsFZbECNKuO8QnJXe5sTx9G15qjPpai2gM5rqN+ENdalVOAlqy3QM/X TSD2KkXnQjtutOYNAMYukSamnvEu1oxH4KoiIK4ThB4OmXtq4Ua5JRDGGZo8Y19M wVHXIHBuHszcZ7zBNhps00yzwE+ymgiAmLkAzZ8XkKJUZf0/p6jrSpoE9aygyDXt +0+3tkreEVIFwCRowq0KXhFaL4+nnQsTqGDVnpkBj70+9WBcVk4zXUPXXYo6ACyK 8PuUQAVP3PGMERCYk5EeXwgGGcIYblRqQtYKxiHHYF5N9zTl+IaEN2wt64DirIRW gIxXIrAIDF5KvcrIsdJJVvPUA2GHKmKQrFSm9lnxiA7abMnFxabBTkGsLgcBlppM XJAJiQI7BBMBAgAlAhsDBgsJCAcDAgQVAggDAxYCAQIeAQIXgAUCUhkTvgUJC0cf hgAKCRC7Vy4OLRgpEHTWD/45wbLJg4iupSCat1+5Le9D84hdgRZIydEjt+cLaSYr sUBZX72P+wDi0wpdcYuiHI+IWlneMQZcBpVuL+ODPek3s6Z4R7XqN9mD79RjaoFy H7870X/y8C4NhjV3UXBOx0o5/1tNtROlqm8PNa1LIKP7mfR61fGo8YmWt6duZxei k/S1WYlR5XqEyUfDaMnID6p8tck1BVxws61DSweYNOgZFWyemO60d2duLEU85L7j vpDIu5q4zSGvnCA5hML1nclc40DFrQsTf13nQsNOojJo+Erc95KyNLp6N9OYt+3M mkBc104XFFyEyHQ6IgfKI834pKnuEh1btsQUJjVwmHsxxqr0YPLsBbdq8fklD5Xr ahqDiMLSVJmm37eqyXGeqtUzgs4i6zMKPbSX3uqR3h/F1uEg9ijhdPAbYXMeQqDR tkHAshp0x+CLCJRZPPvKZHqZBmGeiZg8Rphxd8R8x/KI6gddGlYh/n4MESmZBhMT qSnletjbxmHfcX4Q/M2JmKDpEML7RrPdJTa9Cuc/GdqOr83V/szVEF4cKH3ot520 KQ4F7LmE3XVT084b2Aqhz3Rcp3ubpeRCXsDkV0SGI1qLtJLQ5xvOhaVgi6s1sbX9 i6qOm5UyfdlPwadF//4Lsku/F7cN6qf5asTr+1Pbpc8osZsXuDa0fBzP/gYBxYOv fokCOwQTAQIAJQIbAwYLCQgHAwIEFQIIAwMWAgECHgECF4AFAlXeGLoFCQ8MJIIA CgkQu1cuDi0YKRBXMQ//QiGV6PauMMzFuVDREtLEAzXrSRBDMYLrtIvgTOYMslli mpqpj/DV4XkQsEZY3A5mlWBStGFIG9B7CMAmYveuW/wiDH08ONJ1kEbFs3txFgPj CDRd2a+/U/ALf1J9JnPy1rjAM3HfRvruTx46pxT9CJjRDOyatl911XM0RCbecPZw tMo9em59F1R5rFqxvuP4BbekXdZc1orXfAC/5ot8v+3TK/MD26tjNE8qfAhfLV25 426IoXIG/EoNI7WYkVK3Fiyzjyyq90hF8TI+CLG5mPPx7PLl1LAZaoOYQ5l9J7oL OrYwyDjqcLxGnaSbVa++qooaJnwMusYYJeysV7z+7ZiV2xkxhahMQQs/6i8vtmSW s83pXcmbO3zpXC59qdHmHcMmtXLxM10D7SFXv1GI8Ev5ewKoRIREXNzZeIB+SHdQ A6Ffv6FWAXvITldpR8MeSm3oxFEjLhxhhI6cZoVITw93zUyk9LDq/i583vvdUrnh Uy++ByguXPAuMP+pcseKU5jCxTR+o9K0pwXhRAZC+iBLHLnRInD6rKPOsuIudLBh EqPeqfB5Qec58EeH8Ghz2qkqNIxgZuj1Iu+e1ppteVcXQJIeeBGrmSJaRDJlnh2n F6SWlTIx0ZXYQJ2vMmzn4HvuQiMlKRUHEM8f0DC24YrgEXwjHr3LCZn1oJmq4KmJ AjsEEwECACUFAkqUWzkCGwMFCQeEzgAGCwkIBwMCBBUCCAMDFgIBAh4BAheAAAoJ ELtXLg4tGCkQFq8QAI3o+OEx2feC/0R7qzXe6ANkBEdXDGIFLYggVqtOPS2/C1fY YeZ7y7QCBDy7WGEYin8m+dSLwvKVNOfHKhZ5NLuRTsvC/TylfS4joysZ9NuVFCgp ZlAoZxNndvdCPaX0RdQcgDRkGP9E89qqrglACOJDc2DaEKK2lQ24wK9wwm0lKAHN s3d5qz9TQ2PopBED1LQbyHk29pTy/zJqyKNn0mxD97blvH2mJDU1pISMxNqOv9HA PysjrqRYzrme6Mhy6TfWknWThLhIMDaaOehzMrwTlIYi85bySWYd3DEa7NH3IvWc 6SfLQgaRh93GK7ImY3sI4A2MoIxGVU6CSzHiRfwW1RnOb+3fw0ffvrbV7Qr6axln pF4RQaCXfpDAde8AzIwOGJJUcImdjd4Z2Ji33vmhx3drD2SyYiZ3/3cPjheHBJgH p/LnOFb/ejlvaqIYpFKlIbqsPbl541mhjTsu2yrgfAlRBAIer9m6c0dtjIJnqnXB yNhv11XdiH3rIURCJcKkZIBgMcosun3bMK9hHYQ+A2dvLbWIG4W67xDufYpzmRx8 pLepfQa6FHgD/5a1AJ93fQrkkd0gaDbNt0H/DvQZqLaxaZHKtCVO7JZwPvxw4ttS jq6MW8v78I1D9i/WC53rvUeFTiChm7xtCMzXr08g51Xu/t3q0oOtaHhCHYGOiQI7 BBMBAgAlBQJKlFs5AhsDBQkHhM4ABgsJCAcDAgQVAggDAxYCAQIeAQIXgAAKCRC7 Vy4OLRgpEBavEACN6PjhMdn3gv9Ee6s13ugDZARHVwxiBS2IIFarTj0tvwtX2GHm e8u0AgQ8u1hhGIp/JvnUi8LylTTnxyoWeTS7kU7Lwv08pX0uI6MrGfTblRQoKWZQ KGcTZ3b3Qj2l9EXUHIA0ZBj/RPPaqq4JQAjiQ3Ng2hCitpUNuMCvcMJtJSgBzbN3 eas/U0Nj6KQR//////////////////////////////////////////////////// //////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////w== =1qLu -----END PGP PUBLIC KEY BLOCK----- diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/finalise.d/000077500000000000000000000000001324273057700257545ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/finalise.d/99-cleanup000077500000000000000000000034311324273057700275710ustar00rootroot00000000000000#!/bin/bash if [[ ${DIB_DEBUG_TRACE:-0} -gt 0 ]]; then set -x fi set -eu set -o pipefail if mountpoint -q /dev/shm; then echo "/dev/shm found in /proc/self/mountinfo" elif [[ -k /dev/shm ]]; then echo "/dev/shm exists and is stickied" else fix_shm fi # make world consistant USE="-build" emerge -q --complete-graph=y --jobs=2 --update --newuse --deep --with-bdeps=y @world # rebuild packages that might need it USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild # remove unneeded packages USE="-build" emerge --verbose=n --depclean # rebuild packages that might have somehow depended on the unneeded packages USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild # update config files etc-update --automode -5 # clean up portage files emerge --verbose=n --depclean emaint all -f eselect news read all rm -Rf /root/.ccache/* /usr/src/* /var/cache/edb/dep/* /var/cache/genkernel/* /var/empty/* /var/run/* /var/state/* /var/tmp/* rm -Rf /etc/*- /etc/*.old /etc/ssh/ssh_host_* /root/.*history /root/.lesshst /root/.ssh/known_hosts /root/.viminfo /usr/share/genkernel /usr/lib64/python*/site-packages/gentoolkit/test/eclean/testdistfiles.tar.gz if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then # remove the overlays if [[ ${GENTOO_OVERLAYS} != '' ]]; then for OVERLAY in ${GENTOO_OVERLAYS}; do layman -d "${OVERLAY}" done fi # remove portage files rm -Rf /usr/portage/* /var/cache/portage/distfiles fi # shrink a bit for i in $(find /var/log -type f); do echo > $i; done find /usr/share/man/ -mindepth 1 -maxdepth 1 -path "/usr/share/man/man*" -prune -o -exec rm -rf {} \; # make it so we don't have to reinstall grub if [[ -a /usr/sbin/grub2-install ]]; then mkdir -p /tmp/grub touch /tmp/grub/install fi unfix_shm diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/package-installs.yaml000066400000000000000000000000231324273057700300410ustar00rootroot00000000000000sys-fs/dosfstools: diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/pre-install.d/000077500000000000000000000000001324273057700264145ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-01-migrate-pax000077500000000000000000000003061324273057700325050ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # migrate pt_pax flags to xt_pax if [[ -a /usr/sbin/migrate-pax ]]; then /usr/sbin/migrate-pax -m fi diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-02-flags000077500000000000000000000017711324273057700313730ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # get the directories in order mkdir -p /etc/portage/profile mkdir -p /etc/portage/package.keywords mkdir -p /etc/portage/package.mask mkdir -p /etc/portage/package.unmask mkdir -p /etc/portage/package.use # needed in order to install pip packages as root echo 'dev-python/pip vanilla' >> /etc/portage/package.use/pip # needed to create disk images echo 'sys-fs/lvm2 -thin' >> /etc/portage/package.use/grub echo 'sys-boot/grub device-mapper' >> /etc/portage/package.use/grub # needed in order to install pip packages as root echo '=dev-python/pip-9.0.1-r2 ~amd64' >> /etc/portage/package.keywords/pip if [[ "${GENTOO_PROFILE}" == *"musl"* ]]; then echo "dev-vcs/git -gpg" >> /etc/portage/package.use/musl # gpg doesn't build on musl profiles echo "=sys-block/open-iscsi-2.0.874-r1 ~amd64" >> /etc/portage/package.keywords/musl echo "=sys-block/open-isns-0.97 ~amd64" >> /etc/portage/package.keywords/musl fi 02-gentoo-03-enable-overlays000077500000000000000000000024371324273057700333110ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/pre-install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail if [[ ${GENTOO_OVERLAYS} != '' ]]; then if mountpoint -q /dev/shm; then echo "/dev/shm found in /proc/self/mountinfo" elif [[ -k /dev/shm ]]; then echo "/dev/shm exists and is stickied" else fix_shm fi if [[ ! -f /usr/portage/profiles ]]; then emerge-webrsync -q fi # layman requires cryptography, which needs ecc crypto functions. # binaries providing those functions have questionable legality for # redistribution, so we have to use a version of openssl that works around # it (using fedora's patchset) and also use a version of cryptography that # depends on that version of openssl. echo '=dev-python/cryptography-2.1.3 ~amd64' >> /etc/portage/package.keywords/layman echo '=dev-libs/openssl-1.1.0g-r1 ~amd64' >> /etc/portage/package.keywords/layman echo '=dev-libs/openssl-1.1.0g-r1' >> /etc/portage/package.unmask/layman emerge -q --oneshot --jobs=2 openssl openssh # install layman USE="-build" emerge --deep -q --jobs=2 layman # sync the initial overlay list layman -S # enable the various overlays for OVERLAY in ${GENTOO_OVERLAYS}; do layman -a "${OVERLAY}" done unfix_shm fi 02-gentoo-04-install-desired-python000077500000000000000000000063221324273057700346210ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/pre-install.d#!/bin/bash if [[ ${DIB_DEBUG_TRACE:-0} -gt 0 ]]; then set -x fi set -eu set -o pipefail if mountpoint -q /dev/shm; then echo "/dev/shm found in /proc/self/mountinfo" elif [[ -k /dev/shm ]]; then echo "/dev/shm exists and is stickied" else fix_shm fi if [[ ! -f /usr/portage/profiles ]]; then emerge-webrsync -q fi # python-3.6 and python-3.5 are masked and considered unstable for some reason echo "PYTHON_TARGETS=\"${GENTOO_PYTHON_TARGETS}\"" >> /etc/portage/make.conf if [[ "${GENTOO_PYTHON_TARGETS}" == *"python3_6"* ]]; then echo -e "-python_targets_python3_6\n-python_single_target_python3_6" >> /etc/portage/profile/use.stable.mask echo 'dev-lang/python:3.6 ~amd64' >> /etc/portage/package.keywords/python echo '~dev-python/setuptools-36.0.1 ~amd64' >> /etc/portage/package.keywords/python echo '~dev-python/pyxattr-0.6.0 ~amd64' >> /etc/portage/package.keywords/python echo '~sys-apps/kmod-24 ~amd64' >> /etc/portage/package.keywords/python [[ ! -e /usr/lib64/libpython3.6m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.6 fi [[ "${GENTOO_PYTHON_TARGETS}" == *"python3_5"* ]] && [[ ! -e /usr/lib64/libpython3.5m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.5 [[ "${GENTOO_PYTHON_TARGETS}" == *"python3_4"* ]] && [[ ! -e /usr/lib64/libpython3.4m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.4 [[ "${GENTOO_PYTHON_TARGETS}" == *"python2_7"* ]] && [[ ! -e /usr/lib64/libpython2.7.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:2.7 # disable python in git if we have to, it only supports python-2.7 if [[ "${GENTOO_PYTHON_TARGETS}" != *"python2_7"* ]]; then echo 'dev-vcs/git -python' >> /etc/portage/package.use/git fi # make sure we have the new python for portage bevore we possibly remove python 2.7 # this is done one by one with nodeps because it can sometimes have dependency loops PYTHON_PACKAGES="dev-python/six dev-python/packaging dev-python/pyparsing dev-python/appdirs dev-python/setuptools dev-python/certifi" for PACKAGE in ${PYTHON_PACKAGES}; do USE="-build" emerge -q --oneshot --jobs=2 --with-bdeps=y --update --newuse --deep --nodeps "${PACKAGE}" done USE="-build" emerge -q --oneshot --jobs=2 --with-bdeps=y --update --newuse --deep --complete-graph=y sys-apps/portage dev-python/pyxattr # set the active python version eselect python set ${GENTOO_PYTHON_ACTIVE_VERSION} # allow these uninstalls to fail as they may not be installed to begin with set +e [[ "${GENTOO_PYTHON_TARGETS}" != *"python2_7"* ]] && emerge -C -q dev-lang/python:2.7 [[ "${GENTOO_PYTHON_TARGETS}" != *"python3_4"* ]] && emerge -C -q dev-lang/python:3.4 [[ "${GENTOO_PYTHON_TARGETS}" != *"python3_5"* ]] && emerge -C -q dev-lang/python:3.5 [[ "${GENTOO_PYTHON_TARGETS}" != *"python3_6"* ]] && emerge -C -q dev-lang/python:3.6 set -e # make world consistant USE="-build" emerge -q --complete-graph=y --jobs=2 --update --newuse --deep --with-bdeps=y @world # rebuild packages that might need it USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild # remove unneeded packages USE="-build" emerge --verbose=n --depclean # rebuild packages that might have somehow depended on the unneeded packages USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild unfix_shm diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/root.d/000077500000000000000000000000001324273057700251455ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/root.d/10-gentoo-image000077500000000000000000000100001324273057700276530ustar00rootroot00000000000000#!/bin/bash # Copyright 2016 Matthew Thode # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail [ -n "${ARCH}" ] [ -n "${TARGET_ROOT}" ] if [ 'amd64' = "${ARCH}" ] ; then ARCH='x86_64' fi if ! [ 'x86_64' = "${ARCH}" ] ; then echo "Only x86_64 images are currently available but ARCH is set to ${ARCH}." exit 1 fi # valid gentoo profiles are as follows # default/linux/amd64/13.0 # default/linux/amd64/13.0/no-multilib # hardened/linux/amd64 # hardened/linux/amd64/no-multilib GENTOO_PROFILE=${GENTOO_PROFILE:-'default/linux/amd64/17.0'} if [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.0" ]]; then FILENAME_BASE='gentoo-stage4' SIGNED_SOURCE_SUFFIX='minimal' elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.0/no-multilib" ]]; then FILENAME_BASE='gentoo-stage4-nomultilib' SIGNED_SOURCE_SUFFIX='minimal-nomultilib' elif [[ "${GENTOO_PROFILE}" == "hardened/linux/amd64" ]]; then FILENAME_BASE='gentoo-stage4-hardened' SIGNED_SOURCE_SUFFIX='hardened+minimal' elif [[ "${GENTOO_PROFILE}" == "hardened/linux/amd64/no-multilib" ]]; then FILENAME_BASE='gentoo-stage4-hardened-nomultilib' SIGNED_SOURCE_SUFFIX='hardened+minimal-nomultilib' else echo 'invalid profile, please select from the following profiles' echo 'default/linux/amd64/17.0' echo 'default/linux/amd64/17.0/no-multilib' echo 'hardened/linux/amd64' echo 'hardened/linux/amd64/no-multilib' exit 1 fi DIB_CLOUD_SOURCE=${DIB_CLOUD_SOURCE:-"http://distfiles.gentoo.org/releases/amd64/autobuilds/latest-stage4-amd64-${SIGNED_SOURCE_SUFFIX}.txt"} BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-"http://distfiles.gentoo.org/releases/amd64/autobuilds/$(curl ${DIB_CLOUD_SOURCE} -s | tail -n 1 | cut -d\ -f 1)"} SIGNATURE_FILE="${SIGNATURE_FILE:-${BASE_IMAGE_FILE}.DIGESTS.asc}" CACHED_FILE="${DIB_IMAGE_CACHE}/${FILENAME_BASE}.tar.bz2" CACHED_SIGNATURE_FILE="${DIB_IMAGE_CACHE}/${FILENAME_BASE}.asc" if [ -n "${DIB_OFFLINE}" -a -f "${CACHED_FILE}" ] ; then echo "Not checking freshness of cached ${CACHED_FILE}" else echo 'Fetching Base Image' "${TMP_HOOKS_PATH}"/bin/cache-url "${SIGNATURE_FILE}" "${CACHED_SIGNATURE_FILE}" "${TMP_HOOKS_PATH}"/bin/cache-url "${BASE_IMAGE_FILE}" "${CACHED_FILE}" pushd "${DIB_IMAGE_CACHE}" # import the key # this key can be verified at one of the following places # https://wiki.gentoo.org/wiki/Project:RelEng#Keys # https://dev.gentoo.org/~dolsen/releases/keyrings/gentoo-keys-*.tar.xz # http://distfiles.gentoo.org/distfiles/gentoo-keys-*.tar.xz GPGDIR=$(mktemp -d -t) gpg --no-default-keyring --keyring "${GPGDIR}"/gentookeys.gpg --import "${TMP_HOOKS_PATH}"/extra-data.d/gentoo-releng.gpg # check the sig file gpgv --keyring "${GPGDIR}"/gentookeys.gpg "${CACHED_SIGNATURE_FILE}" if [[ "${?}" != 0 ]]; then echo 'invalid signature file' exit 1 fi rm -rf "${GPGDIR}" echo 'valid key used' CACHED_SHA512SUM=$(grep -A1 -e 'SHA512' "${CACHED_SIGNATURE_FILE}" | grep -e 'tar.bz2$' | cut -d\ -f 1) ACTUAL_SHA512SUM=$(sha512sum "${CACHED_FILE}" | cut -d\ -f 1) if [[ "${ACTUAL_SHA512SUM}" != "${CACHED_SHA512SUM}" ]]; then echo "invalid checksum on downloaded tarball: ${CACHED_FILE}" exit 1 fi echo 'valid checksum' popd fi # Extract the base image (use --numeric-owner to avoid UID/GID mismatch between # image tarball and host OS) sudo tar -C "${TARGET_ROOT}" --numeric-owner --xattrs -xjpf "${CACHED_FILE}" diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/test-elements/000077500000000000000000000000001324273057700265315ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/test-elements/build-succeeds/000077500000000000000000000000001324273057700314245ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/test-elements/build-succeeds/README.rst000066400000000000000000000000641324273057700331130ustar00rootroot00000000000000Test that we can successfully build a gentoo image. diskimage-builder-2.11.0/diskimage_builder/elements/gentoo/test-elements/build-succeeds/element-deps000066400000000000000000000000051324273057700337240ustar00rootroot00000000000000base diskimage-builder-2.11.0/diskimage_builder/elements/growroot/000077500000000000000000000000001324273057700243275ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/growroot/README.rst000066400000000000000000000002461324273057700260200ustar00rootroot00000000000000======== growroot ======== Grow the root partition on first boot. This is only supported on: * ubuntu trusty or later * gentoo * fedora & centos * suse & opensuse diskimage-builder-2.11.0/diskimage_builder/elements/growroot/element-deps000066400000000000000000000000371324273057700266340ustar00rootroot00000000000000dib-init-system install-static diskimage-builder-2.11.0/diskimage_builder/elements/growroot/init-scripts/000077500000000000000000000000001324273057700267575ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/growroot/init-scripts/openrc/000077500000000000000000000000001324273057700302455ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/growroot/init-scripts/openrc/growroot000077500000000000000000000001511324273057700320520ustar00rootroot00000000000000#!/sbin/runscript start() { /usr/local/sbin/growroot eend 0 } depend() { need localmount } diskimage-builder-2.11.0/diskimage_builder/elements/growroot/init-scripts/systemd/000077500000000000000000000000001324273057700304475ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/growroot/init-scripts/systemd/growroot.service000066400000000000000000000003531324273057700337140ustar00rootroot00000000000000[Unit] Description=Grow root partition Wants=systemd-udev-settle.service After=systemd-udev-settle.service [Service] Type=oneshot User=root ExecStart=/usr/local/sbin/growroot RemainAfterExit=true [Install] WantedBy=multi-user.target diskimage-builder-2.11.0/diskimage_builder/elements/growroot/init-scripts/upstart/000077500000000000000000000000001324273057700304615ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/growroot/init-scripts/upstart/growroot.conf000066400000000000000000000002101324273057700332030ustar00rootroot00000000000000description "Growing root partition" console output start on local-filesystems post-start script /usr/local/sbin/growroot end script diskimage-builder-2.11.0/diskimage_builder/elements/growroot/package-installs.yaml000066400000000000000000000000241324273057700304310ustar00rootroot00000000000000growpart: e2fsprogs:diskimage-builder-2.11.0/diskimage_builder/elements/growroot/pkg-map000066400000000000000000000007221324273057700256070ustar00rootroot00000000000000{ "family":{ "redhat": { "growpart": "cloud-utils-growpart", "e2fsprogs": "e2fsprogs" }, "debian":{ "growpart": "cloud-utils", "e2fsprogs": "e2fsprogs" }, "suse": { "growpart": "growpart", "e2fsprogs": "e2fsprogs" }, "gentoo": { "growpart": "sys-fs/growpart", "e2fsprogs": "sys-fs/e2fsprogs" } } } diskimage-builder-2.11.0/diskimage_builder/elements/growroot/post-install.d/000077500000000000000000000000001324273057700272025ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/growroot/post-install.d/80-growroot000077500000000000000000000006671324273057700312500ustar00rootroot00000000000000#!/bin/bash if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail case "$DIB_INIT_SYSTEM" in upstart) # nothing to do exit 0 ;; systemd) systemctl enable growroot.service ;; openrc) # let dib-init-system's postinstall handle enabling init scripts exit 0 ;; *) echo "Unsupported init system" exit 1 ;; esac diskimage-builder-2.11.0/diskimage_builder/elements/growroot/static/000077500000000000000000000000001324273057700256165ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/growroot/static/usr/000077500000000000000000000000001324273057700264275ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/growroot/static/usr/local/000077500000000000000000000000001324273057700275215ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/growroot/static/usr/local/sbin/000077500000000000000000000000001324273057700304545ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/growroot/static/usr/local/sbin/growroot000077500000000000000000000015451324273057700322710ustar00rootroot00000000000000#!/bin/bash # dib-lint: disable=dibdebugtrace set -exu set -o pipefail root_dev=$(df -P / | tail -n 1 | awk '/.*/ { print $1 }') if [ -z "${root_dev}" ]; then echo "*** Root device not found?" exit 1 fi udev_root=$(udevadm info --query=all --name $root_dev) part=$(grep ID_PART_ENTRY_DISK <<< "${udev_root}" | cut -d= -f2) if [ -z "${part}" ]; then echo "*** Root partition not found!" exit 1 fi part_no=$(grep ID_PART_ENTRY_NUMBER <<< "${udev_root}" | cut -d= -f2) if [ -z "${part_no}" ]; then echo "*** Root partition number not found!" exit 1 fi disk=$(find /sys/dev/block -name ${part}) disk="/dev/$(source ${disk}/uevent; echo $DEVNAME)" set +e growpart $disk $part_no # Error code 1 means no change if [ "$?" -le 1 ]; then # always return true because this might not work if were are non ext4 resize2fs $root_dev || true fi diskimage-builder-2.11.0/diskimage_builder/elements/grub2/000077500000000000000000000000001324273057700234665ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/grub2/README.rst000066400000000000000000000004501324273057700251540ustar00rootroot00000000000000===== grub2 ===== This image installs grub2 bootloader on the image, that's necessary for the local boot feature in Ironic to work. This is being made a separated element because usually images have grub2 removed for space reasons and also because they are going to boot from network (PXE boot). diskimage-builder-2.11.0/diskimage_builder/elements/grub2/package-installs.yaml000066400000000000000000000002331324273057700275720ustar00rootroot00000000000000signed_grub_efi: arch: amd64 signed_shim_efi: arch: amd64 grub_bios: arch: amd64 signed_grub_efi_arm64: arch: arm64 grub_bios_arm64: arch: arm64 diskimage-builder-2.11.0/diskimage_builder/elements/grub2/pkg-map000066400000000000000000000012221324273057700247420ustar00rootroot00000000000000{ "family":{ "debian":{ "signed_grub_efi": "grub-efi-amd64-signed", "signed_shim_efi": "shim-signed", "grub_bios": "grub-pc-bin", "signed_grub_efi_arm64": "grub-efi-arm64", "grub_bios_arm64": "grub-efi-arm64-bin" }, "redhat":{ "signed_grub_efi": "efibootmgr grub2-efi-modules grub2-efi", "signed_shim_efi": "shim", "grub_bios": "grub2" }, "suse":{ "signed_grub_efi": "", "signed_shim_efi": "", "grub_bios": "grub2" } }, "default":{ "signed_grub_efi": "grub-efi-amd64-signed", "signed_shim_efi": "shim-signed", "grub_bios": "grub-pc-bin" } } diskimage-builder-2.11.0/diskimage_builder/elements/grub2/post-install.d/000077500000000000000000000000001324273057700263415ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/grub2/post-install.d/01-delete-grubenv000077500000000000000000000003321324273057700314130ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # Delete /boot/grub2/grubenv if it exists because # it will fail while doing grub-install again. rm -f /boot/grub2/grubenv diskimage-builder-2.11.0/diskimage_builder/elements/hpdsa/000077500000000000000000000000001324273057700235445ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/hpdsa/README.rst000066400000000000000000000010511324273057700252300ustar00rootroot00000000000000===== hpdsa ===== This is the hpdsa element. This element enables the 'hpdsa' driver to be included in the ramdisk when invoked during ramdisk/image creation. This driver is required for deploying the HP Proliant Servers Gen9 with Dynamic Smart Array Controllers. Note: This element supports only Ubuntu image/ramdisk to be updated with the hpdsa driver. It installs hp certificate from https://downloads.linux.hp.com/SDR/hpPublicKey2048_key1.pub. Since HP has released this currently only for trusty, It has been restricted to work only for trusty. diskimage-builder-2.11.0/diskimage_builder/elements/hpdsa/package-installs.yaml000066400000000000000000000001211324273057700276440ustar00rootroot00000000000000build-essential: curl: phase: pre-install.d linux-headers-generic: hpdsa-dkms: diskimage-builder-2.11.0/diskimage_builder/elements/hpdsa/post-install.d/000077500000000000000000000000001324273057700264175ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/hpdsa/post-install.d/95-hpdsa000077500000000000000000000007241324273057700277020ustar00rootroot00000000000000#!/bin/bash if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail # Blacklist the ahci driver for hpdsa to recognize the disks. # It can also be done with element "modprobe-blacklist" which # would add the ahci driver to /etc/modprobe.d/blacklist.conf. # To be on safer side, blacklist the driver ahci without the # element "modprobe-blacklist" also. echo "blacklist ahci" >> /etc/modprobe.d/install-hpdsa.conf echo "hpdsa" >> /etc/modules diskimage-builder-2.11.0/diskimage_builder/elements/hpdsa/post-install.d/98-hpdsa-post-dkms000077500000000000000000000003051324273057700316170ustar00rootroot00000000000000#!/bin/bash if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail # This is required for HPDSA driver to be included in the # ramdisk of the image. update-initramfs -u diskimage-builder-2.11.0/diskimage_builder/elements/hpdsa/pre-install.d/000077500000000000000000000000001324273057700262205ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/hpdsa/pre-install.d/06-hpdsa000077500000000000000000000005361324273057700274740ustar00rootroot00000000000000#!/bin/bash if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail # Add hpdsa repository echo 'deb https://downloads.linux.hp.com/SDR/repo/ubuntu-hpdsa' $DIB_RELEASE ' main' \ >> /etc/apt/sources.list.d/ubuntu-hpdsa.list # Add the key curl https://downloads.linux.hp.com/SDR/hpPublicKey2048_key1.pub | apt-key add - diskimage-builder-2.11.0/diskimage_builder/elements/hpdsa/root.d/000077500000000000000000000000001324273057700247515ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/hpdsa/root.d/10-hpdsa-check-distro000077500000000000000000000005751324273057700306000ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail if [ 'ubuntu' != $DISTRO_NAME ] ; then echo "Only ubuntu is supported for hpdsa driver. The DISTRO is set to $DISTRO_NAME" exit 1 fi if [ 'trusty' != $DIB_RELEASE ] ; then echo "Only trusty is supported for hpdsa driver. The DIB_RELEASE is set to $DIB_RELEASE" exit 1 fi diskimage-builder-2.11.0/diskimage_builder/elements/hwburnin/000077500000000000000000000000001324273057700243015ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/hwburnin/README.rst000066400000000000000000000001541324273057700257700ustar00rootroot00000000000000======== hwburnin ======== A hardware test ramdisk - exercises the machine RAM and exercises the hard disks diskimage-builder-2.11.0/diskimage_builder/elements/hwburnin/binary-deps.d/000077500000000000000000000000001324273057700267405ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/hwburnin/binary-deps.d/hwburnin000066400000000000000000000000171324273057700305150ustar00rootroot00000000000000spew memtester diskimage-builder-2.11.0/diskimage_builder/elements/hwburnin/init.d/000077500000000000000000000000001324273057700254665ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/hwburnin/init.d/70-hwburnin000066400000000000000000000001551324273057700274720ustar00rootroot00000000000000echo "Starting HW Burnin" memtester 1G 10 # XXX: Discover RAM size # XXX: Discover and test disks with spew diskimage-builder-2.11.0/diskimage_builder/elements/hwdiscovery/000077500000000000000000000000001324273057700250135ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/hwdiscovery/README.rst000066400000000000000000000022651324273057700265070ustar00rootroot00000000000000=========== hwdiscovery =========== A ramdisk to report the hardware of a machine to an inventory service. This will collect up some basic information about the hardware it boots on: * CPU cores * RAM * Disk * NIC mac address This information will then be collated into a JSON document, base64 encoded and passed, via HTTP POST, to a URL that you must specify on the kernel commandline, thus: HW_DISCOVERY_URL=http://1.2.3.4:56/hw_script.asp This is currently fairly fragile as there can be a huge variability in the number of disks/NICs in servers and how they are configured. If other elements wish to inject data into the hardware discovery data, they can - they should be specified before hwdiscovery to the image building script, and they should contain something like this in their init fragment: _vendor_hwdiscovery_data="$_vendor_hwdiscovery_data \"some vendor key\" : \"some data you care about\", \"some other vendor key\" : \"some other data you care about\"," Note that you are essentially feeding JSON into the main hwdiscovery JSON. This will allow any number of vendor specific hwdiscovery elements to chain together their JSON fragments and maintain consistency. diskimage-builder-2.11.0/diskimage_builder/elements/hwdiscovery/binary-deps.d/000077500000000000000000000000001324273057700274525ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/hwdiscovery/binary-deps.d/hwdiscovery000066400000000000000000000000411324273057700317360ustar00rootroot00000000000000lshw hwinfo dhclient wget base64 diskimage-builder-2.11.0/diskimage_builder/elements/hwdiscovery/element-deps000066400000000000000000000000211324273057700273110ustar00rootroot00000000000000package-installs diskimage-builder-2.11.0/diskimage_builder/elements/hwdiscovery/init.d/000077500000000000000000000000001324273057700262005ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/hwdiscovery/init.d/60-hwdiscovery000066400000000000000000000073221324273057700307200ustar00rootroot00000000000000echo "Starting HW Discovery" function cpu_cores() { hwinfo --cpu | grep -c "Hardware Class: cpu" } function ram() { # XXX: /proc may not be the best place to get this from, but hwinfo reports weird values (e.g. "1GB + 512MB" on a test VM of mine) _KB=$(grep MemTotal /proc/meminfo | awk '{ print $2 }') echo "$((_KB * 1024)) bytes" } function pxe_mac() { local bootif_re='BOOTIF=([^ ]+)' _mac if [[ $(cat /proc/cmdline) =~ $bootif_re ]]; then # If we were booted using pxelinux and its config file has the # IPAPPEND 2 stanza under the entry we booted from, then pxelinux # will have appended a BOOTIF argument to the kernel parameters telling # us what MAC address we are booting with. From that, we can derive the # boot interface with no problems. _mac="${BASH_REMATCH[1]//-/:}" _mac="${_mac#*:}" elif [[ -d /sys/firmware/efi ]] && type efibootmgr &>/dev/null; then # Likewise, if we booted via the network while running in UEFI mode, and # efibootmgr is installed, we can determine the MAC address of the nic we # booted from. It would be good to have code that can also do this using # efivars or parsing the stuff under /sys/firmware/efi/efivars directly, # but that is a trickier thing to do. local -A boot_entries local bootent_re='^Boot([0-9]{4})' local efimac_re='MAC\(([0-9a-f]+)' local k v current_bootent while read line; do k="${line%% *}" v="${line#* }" if [[ $k = BootCurrent:* ]]; then current_bootent="${line##BootCurrent: }" elif [[ $k =~ $bootent_re ]]; then boot_entries["${BASH_REMATCH[1]}"]="$v" fi done < <(efibootmgr -v) if [[ ${boot_entries["$current_bootent"]} =~ $efimac_re ]]; then _mac='' for o in 0 2 4 6 8 10; do _mac+="${BASH_REMATCH[1]:$o:2}:" done _mac=${_mac%:} fi fi if [[ ! $_mac ]]; then # If none of the exact methods worked, fall back on the heuristic # method and just return the mac addresses of all the interfaces # that have a link. Hopefully whatever consumes this info is smarter # than we are. local _info1 _info2 _dev _info1=$(hwinfo --network|grep -B2 "Link detected: yes"|grep -C1 "HW Address:") _info2=$(echo "${_info1}"|awk '/Device File: (vlan*|br*)/{for(x=NR-2;x<=NR+2;x++)d[x];}{a[NR]=$0}END{for(i=1;i<=NR;i++)if(!(i in d))print a[i]}') _dev=$(echo "${_info1}" | grep "Device File:"|awk -F':' {'print $2'}|tr -d ' ') _mac=$(echo "${_info2}" | grep "HW Address:"|awk -F'ss:' {'print $2'}|tr -d ' ') fi echo $_mac export HW_DISCOVERY_BOOT_IFACE="$_mac" } function disk() { # XXX: This is returning only the first disk discovered, which is very likely to be insufficient on machines that present us with multiple disks # XXX: This is likely reporting in TB, but the units are not guaranteed. Maybe convert to bytes? lshw -C disk | grep size | awk -F'(' '{ print $2 }' | tr -d ')' | head -1 } function raw_disk() { hwinfo --disk } function raw_network() { hwinfo --network } HW_DISCOVERY_OUTPUT=$(cat <&2 exit 0 fi if [ ! -d "${DIB_ILO_FIRMWARE_PATH}" ]; then echo "Firmware path not found: ${DIB_ILO_FIRMWARE_PATH}" exit 1 fi # We copy to /tmp because its temporary - we do a mv later into the initramfs # image. mkdir -p "$TMP_MOUNT_PATH/tmp/ilo/" find "${DIB_ILO_FIRMWARE_PATH}" -type f \( -name '*.scexe' -o -name '*.exe' \) | while read FWNAME do TARGET="$TMP_MOUNT_PATH/tmp/ilo/$(basename "$FWNAME")" echo Unpacking $FWNAME to $TARGET bash "$FWNAME" "--unpack=$TARGET" echo Done done diskimage-builder-2.11.0/diskimage_builder/elements/ilo/init.d/000077500000000000000000000000001324273057700244155ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ilo/init.d/50-ilo-firmware000066400000000000000000000013351324273057700271610ustar00rootroot00000000000000if [ -d /ilo ]; then find /ilo -type f \( -name flash_ilo3 -o -name flash_ilo4 \) | while read FWNAME do cd "$(dirname "$FWNAME")" logname="$(basename "$(dirname "$FWNAME")").log" echo Running $FWNAME "$FWNAME" --silent --log "$logname" result=$? echo Finished $FWNAME with result $? cat "$logname" #avoid leaking space during large runs rm "$logname" if [ $result -eq 2 ]; then echo "Firmware already up to date" elif [ $result -eq 4 ]; then echo "Firmware not applicable to machine" elif [ $result -ne 0 ]; then echo "Error applying firmware" troubleshoot fi done fi diskimage-builder-2.11.0/diskimage_builder/elements/ilo/ramdisk-install.d/000077500000000000000000000000001324273057700265505ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ilo/ramdisk-install.d/50-ilo-firmware000077500000000000000000000002451324273057700313160ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail if [ ! -d /tmp/ilo ]; then exit 0 fi mv /tmp/ilo "$TMP_MOUNT_PATH" diskimage-builder-2.11.0/diskimage_builder/elements/install-bin/000077500000000000000000000000001324273057700246615ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/install-bin/README.rst000066400000000000000000000002171324273057700263500ustar00rootroot00000000000000=========== install-bin =========== Add any files in an element's bin directory to /usr/local/bin in the created image with permissions 0755. diskimage-builder-2.11.0/diskimage_builder/elements/install-bin/pre-install.d/000077500000000000000000000000001324273057700273355ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/install-bin/pre-install.d/01-install-bin000077500000000000000000000002431324273057700317140ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail install -m 0755 -o root -g root $(dirname $0)/../bin/* /usr/local/bin diskimage-builder-2.11.0/diskimage_builder/elements/install-static/000077500000000000000000000000001324273057700254005ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/install-static/README.rst000066400000000000000000000005361324273057700270730ustar00rootroot00000000000000============== install-static ============== Copy static files into the built image. The contents of any ``static/`` subdirs of elements will be installed into these images at build time using ``rsync -lCr``. So to install a file ``/etc/boo``, include ``static/etc/boo`` in your element. Note: This installs all files with owner and group of root. diskimage-builder-2.11.0/diskimage_builder/elements/install-static/element-deps000066400000000000000000000000211324273057700276760ustar00rootroot00000000000000package-installs diskimage-builder-2.11.0/diskimage_builder/elements/install-static/install.d/000077500000000000000000000000001324273057700272705ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/install-static/install.d/10-install-static-files000077500000000000000000000006461324273057700334750ustar00rootroot00000000000000#!/bin/bash # Note that this relies on the detail that all elements share one dir inside # the chroot. This will copy all the files that elements have added to # element/static into the image. Mode, symlinks etc will be respected. if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail STATIC_SOURCE="$(dirname $0)/../static" [ -d "${STATIC_SOURCE}" ] || exit 0 rsync -lCr "${STATIC_SOURCE}"/ / diskimage-builder-2.11.0/diskimage_builder/elements/install-static/package-installs.yaml000066400000000000000000000000071324273057700315030ustar00rootroot00000000000000rsync: diskimage-builder-2.11.0/diskimage_builder/elements/install-types/000077500000000000000000000000001324273057700252555ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/install-types/README.rst000066400000000000000000000033231324273057700267450ustar00rootroot00000000000000============= install-types ============= Enable install-types support for elements. Install types permit elements to be installed from different sources, such as git repositories, distribution packages, or pip. The default install type is 'source' but it can be modified on the disk-image-create command line via the --install-type option. For example you can set: --install-type=package to enable package installs by default. Alternately, you can also set DIB\_DEFAULT\_INSTALLTYPE. Many elements expose different install types. The different implementations live under `--install` directories under an element's install.d. The base element enables the chosen install type by symlinking the correct hook scripts under install.d directly. `` can be a string of alphanumeric and '-' characters, but typically corresponds to the element name. For example, the nova element would provide: nova/install.d/nova-package-install/74-nova nova/install.d/nova-source-install/74-nova The following symlink would be created for the package install type: install.d/74-nova -> nova-package-install/74-nova Or, for the source install type: install.d/74-nova -> nova-source-install/74-nova All other scripts that exist under install.d for an element will be executed as normal. This allows common install code to live in a script under install.d. Environment Variables --------------------- DIB_INSTALLTYPE_ :Required: No :Default: source :Description: Set the install type for the dir prefix. '-' characters can be replaced with a '_'. :Example: ``DIB_INSTALLTYPE_simple_init=repo`` Sets the simple-init element install type to be repo. diskimage-builder-2.11.0/diskimage_builder/elements/install-types/extra-data.d/000077500000000000000000000000001324273057700275315ustar00rootroot0000000000000099-enable-install-types000077500000000000000000000045231324273057700336770ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/install-types/extra-data.d#!/bin/bash # Copyright 2014 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail declare -a SPECIFIED_ELEMS # Add a blank first element to the array, since we're running with -u, we don't # want an unbound variable error if no element install types were specified. SPECIFIED_ELEMS[0]="" # List of all env vars to set install types PREFIX="DIB_INSTALLTYPE_" INSTALL_TYPE_VARS=$(env | grep ^$PREFIX | cut -d'=' -f1 || echo "") for _install_type_var in $INSTALL_TYPE_VARS; do # Get the element name and its install type INSTALLDIRPREFIX=${_install_type_var:${#PREFIX}} INSTALLDIRPREFIX=${INSTALLDIRPREFIX//_/-} eval INSTALLTYPE=\$$_install_type_var # Create symlink for correct install type pushd $TMP_HOOKS_PATH/install.d if [ -d $INSTALLDIRPREFIX-$INSTALLTYPE-install ]; then ln -sf $INSTALLDIRPREFIX-$INSTALLTYPE-install/* . fi popd SPECIFIED_ELEMS+=($INSTALLDIRPREFIX) done # For any existing *--install directory under install.d, # if an environment variable setting a different install type was not seen, # enable the default (set via --install-type). default_install_type_dirs=$(find $TMP_HOOKS_PATH/install.d -maxdepth 1 -name "*-${DIB_DEFAULT_INSTALLTYPE}-install" -type d) for _install_dir in $default_install_type_dirs; do SUFFIX="-${DIB_DEFAULT_INSTALLTYPE}-install" _install_dir=$(basename $_install_dir) INSTALLDIRPREFIX=${_install_dir%$SUFFIX} found=0 for specified in ${SPECIFIED_ELEMS[@]}; do if [ "$specified" = "$INSTALLDIRPREFIX" ]; then found=1 break fi done # install type not specified, assume source if [ "$found" = "0" ]; then pushd $TMP_HOOKS_PATH/install.d ln -sf $_install_dir/* . popd fi done diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/000077500000000000000000000000001324273057700250245ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/README.rst000066400000000000000000000036411324273057700265170ustar00rootroot00000000000000============ ironic-agent ============ Builds a ramdisk with ironic-python-agent. More information can be found at: https://git.openstack.org/cgit/openstack/ironic-python-agent/ Beyond installing the ironic-python-agent, this element does the following: * Installs the ``dhcp-all-interfaces`` so the node, upon booting, attempts to obtain an IP address on all available network interfaces. * Disables the ``iptables`` service on SysV and systemd based systems. * Disables the ``ufw`` service on Upstart based systems. * Installs packages required for the operation of the ironic-python-agent:: ``qemu-utils`` ``parted`` ``hdparm`` ``util-linux`` ``genisoimage`` * When installing from source, ``python-dev`` and ``gcc`` are also installed in order to support source based installation of ironic-python-agent and its dependencies. * Install the certificate if any, which is set to the environment variable ``DIB_IPA_CERT`` for validating the authenticity by ironic-python-agent. The certificate can be self-signed certificate or CA certificate. * Compresses initramfs with command specified in environment variable ``DIB_IPA_COMPRESS_CMD``, which is 'gzip' by default. This command should listen for raw data from stdin and write compressed data to stdout. Command can be with arguments. This element outputs three files: - ``$IMAGE-NAME.initramfs``: The deploy ramdisk file containing the ironic-python-agent (IPA) service. - ``$IMAGE-NAME.kernel``: The kernel binary file. - ``$IMAGE-NAME.vmlinuz``: A hard link pointing to the ``$IMAGE-NAME.kernel`` file; this is just a backward compatibility layer, please do not rely on this file. .. note:: The package based install currently only enables the service when using the systemd init system. This can easily be changed if there is an agent package which includes upstart or sysv packaging. .. note:: Using the ramdisk will require at least 1.5GB of ram diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/cleanup.d/000077500000000000000000000000001324273057700266755ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/cleanup.d/99-ramdisk-create000077500000000000000000000035321324273057700317600ustar00rootroot00000000000000#!/bin/bash # dib-lint: disable=safe_sudo if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail [ -n "$TARGET_ROOT" ] USER=${USER:-$(whoami)} source $_LIB/img-functions IMAGE_PATH=$(readlink -f $IMAGE_NAME) cd $TARGET_ROOT DIB_IPA_COMPRESS_CMD="${DIB_IPA_COMPRESS_CMD:-gzip}" echo "#disabled" > ./tmp/fstab.new sudo mv ./tmp/fstab.new ./etc/fstab sudo ln -s ./sbin/init ./ # Note: The pci.ids, which is used by lshw, locate on Ubuntu # in /usr/share/misc. Therefore we are removing only the # ./usr/share/misc/m* (will remove the magic and magic.mgc files). # on RHEL pci.ids is locate on /usr/share/hwdata/pci.ids. sudo find . -xdev \ -path './sys/*' -prune -o \ -path './tmp/*' -prune -o \ -path './boot/*' -prune -o \ -path './root/.cache' -prune -o \ -path './usr/include/*' -prune -o \ -path './usr/lib/locale/*' -prune -o \ -path './usr/share/doc/*' -prune -o \ -path './usr/share/man/*' -prune -o \ -path './usr/share/info/*' -prune -o \ -path './usr/share/licenses/*' -prune -o \ -path './usr/share/misc/m*' -prune -o \ -path './usr/src/kernels/*' -prune -o \ -path './var/cache/*' -prune -o \ -name '*.pyc' -prune -o \ -name '*.pyo' -prune -o \ -print | sudo cpio -o -H newc | ${DIB_IPA_COMPRESS_CMD} > ${IMAGE_PATH}.initramfs select_boot_kernel_initrd $TARGET_ROOT sudo cp $BOOTDIR/$KERNEL ${IMAGE_PATH}.kernel sudo chown $USER: ${IMAGE_PATH}.kernel # TODO(lucasagomes): Create a hard link for the .vmlinuz file to keep # it backward compatible. Remove it after it has been consistent and # documented in both places for at least one full OpenStack release cycle echo "WARNING: The kernel extension .vmlinuz has been deprecated. Please rely on the file with the extension .kernel instead." sudo rm -f ${IMAGE_PATH}.vmlinuz ln ${IMAGE_PATH}.kernel ${IMAGE_PATH}.vmlinuz diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/element-deps000066400000000000000000000002201324273057700273230ustar00rootroot00000000000000dhcp-all-interfaces install-static no-final-image package-installs pip-and-virtualenv pkg-map runtime-ssh-host-keys source-repositories svc-map diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/element-provides000066400000000000000000000000241324273057700302250ustar00rootroot00000000000000ironic-python-agent diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/environment.d/000077500000000000000000000000001324273057700276125ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/environment.d/01-ironic-agent.sh000066400000000000000000000004671324273057700327520ustar00rootroot00000000000000# backwards compatibility with the previous environment # variable for the ironic-agent source repository if [ -n "${DIB_REPOREF_agent:-}" ]; then echo "WARNING: DIB_REPOREF_agent is deprecated. Please update to use DIB_REPOREF_ironic_agent instead." export DIB_REPOREF_ironic_agent=${DIB_REPOREF_agent} fi diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/extra-data.d/000077500000000000000000000000001324273057700273005ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/extra-data.d/15-ssl-ca-copy000077500000000000000000000002771324273057700316110ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail if [ -n "${DIB_IPA_CERT:=}" ]; then cp $DIB_IPA_CERT $TMP_HOOKS_PATH/ipa-trusted-cert.pem fi diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/finalise.d/000077500000000000000000000000001324273057700270405ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/finalise.d/99-remove-extra-packages000077500000000000000000000013631324273057700334220ustar00rootroot00000000000000#!/bin/bash if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail # TODO(lucasagomes): optimize the ramdisk for other OSs if [ $DISTRO_NAME = 'fedora' ] ; then _remove_yum='' if [ $DIB_RELEASE -ge 22 ]; then # if we are on F22, we can remove yum if there, because it has # been superseeded by dnf _remove_yum='yum' fi install-packages -e kernel-debug-devel gcc fedora-logos \ rsync pykickstart \ genisoimage tcpdump \ man-db kbd-misc \ plymouth cronie ${_remove_yum} ${YUM:-yum} clean all # Rebuilding the rpm database after removing packages will reduce # its size rpm --rebuilddb fi diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/install.d/000077500000000000000000000000001324273057700267145ustar00rootroot00000000000000ironic-agent-package-install/000077500000000000000000000000001324273057700342515ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/install.dpackage-installs-ironic-agent000066400000000000000000000000241324273057700417670ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-package-installironic-python-agent ironic-agent-source-install/000077500000000000000000000000001324273057700341565ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/install.d60-ironic-agent-install000077500000000000000000000032441324273057700403550ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install#!/bin/bash if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail SCRIPTDIR=$(dirname $0) IPADIR=/usr/share/ironic-python-agent # Generate upper-constraints $IPADIR/imagebuild/common/generate_upper_constraints.sh $IPADIR/upper-constraints.txt # create the virtual environment virtualenv $IPADIR/venv # pip might be an older version which does not support the -c option, therefore upgrade first $IPADIR/venv/bin/pip install pip --upgrade # install IPA inside the virtual environment $IPADIR/venv/bin/pip install -c $IPADIR/upper-constraints.txt $IPADIR # FIXME(lucasagomes): Figure out how we can use the "--install-option" # parameter for pip install so we don't have to manually create a symlink # create the launcher link so services can use it ln -s $IPADIR/venv/bin/ironic-python-agent /usr/local/bin/ironic-python-agent case "$DIB_INIT_SYSTEM" in upstart) install -D -g root -o root -m 0755 ${SCRIPTDIR}/ironic-python-agent.conf /etc/init/ironic-python-agent.conf ;; systemd) install -D -g root -o root -m 0755 ${SCRIPTDIR}/ironic-python-agent.service /usr/lib/systemd/system/ironic-python-agent.service ;; sysv) install -D -g root -o root -m 0755 ${SCRIPTDIR}/ironic-python-agent.init /etc/init.d/ironic-python-agent.init update-rc.d ironic-python-agent.init defaults ;; *) echo "Unsupported init system" exit 1 ;; esac # Copying the self signed certificate for request library if [ -f /tmp/in_target.d/ipa-trusted-cert.pem ]; then cat /tmp/in_target.d/ipa-trusted-cert.pem >> $($IPADIR/venv/bin/python -c "import requests; print requests.certs.where()") fi ironic-python-agent.conf000066400000000000000000000006001324273057700407170ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install# ironic-python-agent - OpenStack Ironic Python Agent # # The ironic-python-agent helps ironic in deploying instances. description "Ironic Python Agnet" start on runlevel [2345] stop on runlevel [!2345] respawn respawn limit 10 5 umask 022 expect stop console output pre-start script echo Starting Ironic Python Agent end script exec /usr/local/bin/ironic-python-agent ironic-python-agent.init000077500000000000000000000011741324273057700407470ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install#!/bin/sh -e ### BEGIN INIT INFO # Provides: ironic-python-agent # Required-Start: $local_fs networking # Required-Stop: $local_fs # Default-Start: S # Default-Stop: 0 6 # X-Start-Before: # Short-Description: Ironic Python Agent # Description: Starts Ironic Python Agent for instance deployment ### END INIT INFO NAME=ironic-python-agent INIT_NAME=/etc/init.d/${NAME} SCRIPT_NAME=/usr/local/bin/${NAME} [ -x $SCRIPT_NAME ] || exit 0 case "$1" in start) $SCRIPT_NAME ;; stop) ;; *) echo "Usage: $INIT_NAME {start|stop}" exit 1 ;; esac exit 0 ironic-python-agent.service000066400000000000000000000003411324273057700414340ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install[Unit] Description=Ironic Python Agent After=network-online.target [Service] ExecStartPre=/sbin/modprobe vfat ExecStart=/usr/local/bin/ironic-python-agent Restart=always RestartSec=30s [Install] WantedBy=multi-user.target diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/package-installs.yaml000066400000000000000000000003531324273057700311330ustar00rootroot00000000000000tgt: curl: # dmidecode does not exist for ppc* arches so we use lshw dmidecode: not-arch: ppc64,ppc64el,ppc64le lshw: ipmitool: qemu-utils: gcc: python-dev: parted: hdparm: util-linux: genisoimage: gdisk: kmod: psmisc: dosfstools: diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/pkg-map000066400000000000000000000007331324273057700263060ustar00rootroot00000000000000{ "family": { "redhat": { "tgt": "scsi-target-utils", "curl": "curl", "ironic-python-agent": "openstack-ironic-python-agent", "qemu-utils": "qemu-img", "python-dev": "python2-devel" }, "debian": { "tgt": "tgt", "curl": "curl" } }, "default": { "tgt": "tgt", "curl": "curl", "ironic-python-agent": "openstack-ironic-python-agent", "qemu-utils": "qemu-utils", "python-dev": "python-dev" } } diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/post-install.d/000077500000000000000000000000001324273057700276775ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/post-install.d/80-ironic-agent000077500000000000000000000015531324273057700324350ustar00rootroot00000000000000#!/bin/bash if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail install-packages -e git cloud-init rm -rf /tmp/ironic-python-agent case "$DIB_INIT_SYSTEM" in upstart) if [ -f /etc/init/ufw.conf ]; then mv /etc/init/ufw.conf /etc/init/ufw.conf.disabled fi if [ -f /etc/init/tgt.conf ]; then mv /etc/init/tgt.conf /etc/init/tgt.conf.disabled fi ;; systemd) if [[ $(systemctl --no-pager list-unit-files iptables) =~ 'enabled' ]]; then systemctl disable iptables.service fi systemctl enable $(svc-map ironic-python-agent).service systemctl enable ironic-agent-create-rescue-user.path ;; sysv) update-rc.d iptables disable ;; *) echo "Unsupported init system" exit 1 ;; esac diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/source-repository-ironic-agent000066400000000000000000000001501324273057700330350ustar00rootroot00000000000000ironic-agent git /usr/share/ironic-python-agent https://git.openstack.org/openstack/ironic-python-agent diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/static/000077500000000000000000000000001324273057700263135ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/static/etc/000077500000000000000000000000001324273057700270665ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/static/etc/systemd/000077500000000000000000000000001324273057700305565ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/static/etc/systemd/system/000077500000000000000000000000001324273057700321025ustar00rootroot00000000000000ironic-agent-create-rescue-user.path000066400000000000000000000002421324273057700407570ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/static/etc/systemd/system[Unit] Description=Ironic user rescue - notify path existence [Path] PathExists=/etc/ipa-rescue-config/ipa-rescue-password [Install] WantedBy=multi-user.target ironic-agent-create-rescue-user.service000066400000000000000000000003241324273057700414640ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/static/etc/systemd/system[Unit] Description=Ironic agent rescue user creation [Service] ExecStart=/bin/bash /usr/local/bin/ironic-python-agent-create-rescue-user.sh Type=oneshot RemainAfterExit=yes [Install] WantedBy=multi-user.target diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/static/usr/000077500000000000000000000000001324273057700271245ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/static/usr/local/000077500000000000000000000000001324273057700302165ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/static/usr/local/bin/000077500000000000000000000000001324273057700307665ustar00rootroot00000000000000ironic-python-agent-create-rescue-user.sh000077500000000000000000000004671324273057700406540ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/static/usr/local/bin#!/bin/bash if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail echo "Adding rescue user with root privileges..." crypted_pass=$( /etc/sudoers.d/rescue diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/svc-map000066400000000000000000000002031324273057700263100ustar00rootroot00000000000000ironic-python-agent: default: openstack-ironic-python-agent redhat: openstack-ironic-python-agent debian: ironic-python-agentdiskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/test-elements/000077500000000000000000000000001324273057700276155ustar00rootroot00000000000000build-succeeds-fedora/000077500000000000000000000000001324273057700336675ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/test-elementselement-deps000066400000000000000000000000441324273057700361720ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/test-elements/build-succeeds-fedorafedora-minimal openstack-ci-mirrors element-type000066400000000000000000000000101324273057700362110ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ironic-agent/test-elements/build-succeeds-fedoraramdisk diskimage-builder-2.11.0/diskimage_builder/elements/iso/000077500000000000000000000000001324273057700232375ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/iso/README.rst000066400000000000000000000035611324273057700247330ustar00rootroot00000000000000=== iso === Generates a bootable ISO image from the kernel/ramdisk generated by the elements ``baremetal``, ``ironic-agent`` or ``ramdisk``. It uses isolinux to boot on BIOS machines and grub to boot on EFI machines. This element has been tested on the following distro(s): * ubuntu * fedora * debian **NOTE**: For other distros, please make sure the ``isolinux.bin`` file exists at ``/usr/lib/syslinux/isolinux.bin``. baremetal element ----------------- When used with ``baremetal`` element, this generates a bootable ISO image named ``-boot.iso`` booting the generated kernel and ramdisk. It also automatically appends kernel command-line argument 'root=UUID='. Any more kernel command-line arguments required may be provided by specifying them in ``DIB_BOOT_ISO_KERNEL_CMDLINE_ARGS``. **NOTE**: It uses pre-built efiboot.img by default to work for UEFI machines. This is because of a bug in latest version of grub[1]. The user may choose to avoid using pre-built binary and build efiboot.img on their own machine by setting the environment variable DIB\_UEFI\_ISO\_BUILD\_EFIBOOT to 1 (this might work only on certain versions of grub). The current efiboot.img was generated by the method build\_efiboot\_img() in 100-build-iso on Ubuntu 13.10 with grub 2.00-19ubuntu2.1. ramdisk element --------------- When used with ``ramdisk`` element, this generates a bootable ISO image named ``.iso`` booting the generated kernel and ramdisk. It also automatically appends kernel command-line argument 'boot\_method=vmedia' which is required for Ironic drivers ``iscsi_ilo``. ironic-agent element -------------------- When used with ``ironic-agent`` element, this generates a bootable ISO image named ``.iso`` which boots the agent kernel and agent ramdisk. **REFERENCES** [1] https://bugs.launchpad.net/ubuntu/+source/grub2/+bug/1378658 diskimage-builder-2.11.0/diskimage_builder/elements/iso/cleanup.d/000077500000000000000000000000001324273057700251105ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/iso/cleanup.d/100-build-iso000077500000000000000000000137001324273057700272240ustar00rootroot00000000000000#!/bin/bash # # Copyright 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # dib-lint: disable=safe_sudo if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail function build_efiboot_img() { # Signed bootloader files are saved at tmp/bootloader_files during # post-install stage needs to copy them into image. # Signed bootloader packages used are:- # Ubuntu: grub-efi-amd64-signed and shim-signed # Fedora: grub2-efi and shim if [ $DISTRO_NAME = "fedora" ]; then cp $TMP_BOOTLOADER_DIR/shim.efi $TMP_BUILD_DIR/bootx64.efi cp $TMP_BOOTLOADER_DIR/grubx64.efi $TMP_BUILD_DIR/grubx64.efi else cp $TMP_BOOTLOADER_DIR/shim.efi.signed $TMP_BUILD_DIR/bootx64.efi cp $TMP_BOOTLOADER_DIR/grubx64.efi.signed $TMP_BUILD_DIR/grubx64.efi fi dd if=/dev/zero of=$TMP_BUILD_DIR/efiboot.img bs=1K count=5760 mkdosfs -F 12 $TMP_BUILD_DIR/efiboot.img # Create a temporary mount point: MOUNTPOINT=$TMP_BUILD_DIR/tmpmount mkdir $MOUNTPOINT sudo mount -o loop $TMP_BUILD_DIR/efiboot.img $MOUNTPOINT sudo mkdir -p $MOUNTPOINT/EFI/BOOT sudo cp $TMP_BUILD_DIR/bootx64.efi $MOUNTPOINT/EFI/BOOT sudo cp $TMP_BUILD_DIR/grubx64.efi $MOUNTPOINT/EFI/BOOT sudo umount $MOUNTPOINT rmdir $MOUNTPOINT cp $TMP_BUILD_DIR/efiboot.img $TMP_IMAGE_DIR/isolinux } function build_iso() { KERNEL=$1 INITRD=$2 KERNEL_CMDLINE_ARGS=$3 OUTPUT_FILENAME=$4 SCRIPTNAME=$(basename $0) SCRIPTDIR=$(dirname $0) MKISOFS="/usr/bin/mkisofs" if [ $DISTRO_NAME = "fedora" ]; then EFI_BOOT_DIR="EFI/fedora" EXTRA_KERNEL_PARAMS="usbcore.autosuspend=-1" #debian elif [ $DISTRO_NAME = "debian" ]; then EFI_BOOT_DIR="EFI/debian" EXTRA_KERNEL_PARAMS="" #ubuntu: else EFI_BOOT_DIR="EFI/ubuntu" EXTRA_KERNEL_PARAMS="" fi # Create a temporary build directory for holding the contents of iso TMP_IMAGE_DIR="$TMP_BUILD_DIR/image" echo "Creating temporary directory $TMP_IMAGE_DIR" mkdir -p "$TMP_IMAGE_DIR" # Copy isolinux bin to the isolinux directory mkdir -p "$TMP_IMAGE_DIR/isolinux" echo "Copying isolinux.bin" ISOLINUX_BIN=$TMP_BOOTLOADER_DIR/isolinux.bin LDLINUX_BIN=$TMP_BOOTLOADER_DIR/ldlinux.c32 cp $ISOLINUX_BIN "$TMP_IMAGE_DIR/isolinux" # Starting from SYSLINUX 5.00, the isolinux.bin is dependent # on ldlinux.c32. # http://www.syslinux.org/wiki/index.php/Library_modules if [ -f "$LDLINUX_BIN" ]; then cp $LDLINUX_BIN "$TMP_IMAGE_DIR/isolinux" fi # Copy initrd, kernel echo "Copying kernel to $TMP_IMAGE_DIR/vmlinuz" cp $KERNEL "$TMP_IMAGE_DIR/vmlinuz" echo "Copying initrd to $TMP_IMAGE_DIR/initrd" cp $INITRD "$TMP_IMAGE_DIR/initrd" # Generate isolinux.cfg for default booting echo "Generating isolinux.cfg" cat > "$TMP_IMAGE_DIR/isolinux/isolinux.cfg" << END_CONFIG DEFAULT install LABEL install menu label "Install image" kernel /vmlinuz append initrd=/initrd $KERNEL_CMDLINE_ARGS -- TIMEOUT 5 PROMPT 0 END_CONFIG echo "Creating EFI/ubuntu directory" mkdir -p "$TMP_IMAGE_DIR/$EFI_BOOT_DIR" # Generate grub.cfg for default booting echo "Generating grub.cfg" cat > "$TMP_IMAGE_DIR/$EFI_BOOT_DIR/grub.cfg" << END_CONFIG set default="0" set timeout="5" set hidden_timeout_quiet=false menuentry "install" { search --set=root --label VMEDIA_BOOT_ISO linuxefi /vmlinuz $EXTRA_KERNEL_PARAMS $KERNEL_CMDLINE_ARGS -- initrdefi /initrd } END_CONFIG build_efiboot_img # Create the ISO echo "Generating the ISO" $MKISOFS -r -V "VMEDIA_BOOT_ISO" -cache-inodes -J -l \ -b isolinux/isolinux.bin -no-emul-boot \ -boot-load-size 4 -boot-info-table \ -eltorito-alt-boot -e isolinux/efiboot.img \ -no-emul-boot -o $OUTPUT_FILENAME $TMP_IMAGE_DIR } IMAGE_NAME=${IMAGE_NAME:-'image'} if echo $IMAGE_ELEMENT | grep -q '\bramdisk\b'; then EMITTED_KERNEL=$IMAGE_NAME.kernel EMITTED_RAMDISK=$IMAGE_NAME.initramfs DIB_DEPLOY_ISO_KERNEL_CMDLINE_ARGS=${DIB_DEPLOY_ISO_KERNEL_CMDLINE_ARGS:-} EMITTED_KERNEL_CMDLINE_ARGS="boot_method=vmedia $DIB_DEPLOY_ISO_KERNEL_CMDLINE_ARGS" EMITTED_ISO_FILENAME=$IMAGE_NAME.iso elif echo $IMAGE_ELEMENT | grep -q '\bironic-agent\b'; then EMITTED_KERNEL=$IMAGE_NAME.vmlinuz EMITTED_RAMDISK=$IMAGE_NAME.initramfs DIB_DEPLOY_ISO_KERNEL_CMDLINE_ARGS=${DIB_DEPLOY_ISO_KERNEL_CMDLINE_ARGS:-} EMITTED_KERNEL_CMDLINE_ARGS="boot_method=vmedia $DIB_DEPLOY_ISO_KERNEL_CMDLINE_ARGS" EMITTED_ISO_FILENAME=$IMAGE_NAME.iso elif echo $IMAGE_ELEMENT | grep -q '\bbaremetal\b'; then EMITTED_KERNEL=${IMAGE_NAME}.vmlinuz EMITTED_RAMDISK=${IMAGE_NAME}.initrd EMITTED_KERNEL_CMDLINE_ARGS="root=UUID=$DIB_IMAGE_ROOT_FS_UUID" DIB_BOOT_ISO_KERNEL_CMDLINE_ARGS=${DIB_BOOT_ISO_KERNEL_CMDLINE_ARGS:-} if [ -n "$DIB_BOOT_ISO_KERNEL_CMDLINE_ARGS" ]; then EMITTED_KERNEL_CMDLINE_ARGS="$EMITTED_KERNEL_CMDLINE_ARGS $DIB_BOOT_ISO_KERNEL_CMDLINE_ARGS" fi export EMITTED_ISO_FILENAME="$IMAGE_NAME-boot.iso" else echo "Cannot find the kernel/ramdisk to build the iso image. " echo "Please use 'iso' element with either 'baremetal' or 'ramdisk' elements" fi export TMP_BOOTLOADER_DIR=$TARGET_ROOT/tmp/bootloader_files build_iso "$EMITTED_KERNEL" "$EMITTED_RAMDISK" "$EMITTED_KERNEL_CMDLINE_ARGS" \ "$EMITTED_ISO_FILENAME" # Clean-up the bootloaders directory sudo rm -fr $TMP_BOOTLOADER_DIR diskimage-builder-2.11.0/diskimage_builder/elements/iso/element-deps000066400000000000000000000000271324273057700255430ustar00rootroot00000000000000grub2 package-installs diskimage-builder-2.11.0/diskimage_builder/elements/iso/package-installs.yaml000066400000000000000000000000451324273057700273440ustar00rootroot00000000000000syslinux: syslinux-common: isolinux: diskimage-builder-2.11.0/diskimage_builder/elements/iso/pkg-map000066400000000000000000000007201324273057700245150ustar00rootroot00000000000000{ "distro":{ "debian":{ "syslinux": "syslinux", "syslinux-common": "syslinux-common", "isolinux": "isolinux" } }, "family":{ "debian":{ "syslinux": "syslinux", "syslinux-common": "", "isolinux": "" }, "redhat":{ "syslinux": "", "syslinux-common": "", "isolinux": "" } }, "default":{ "syslinux": "syslinux", "syslinux-common": "", "isolinux": "" } } diskimage-builder-2.11.0/diskimage_builder/elements/iso/post-install.d/000077500000000000000000000000001324273057700261125ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/iso/post-install.d/01-copy-bootloaders000077500000000000000000000022731324273057700315470ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eux set -o pipefail TMP_BOOTLOADER_DIR=/tmp/bootloader_files mkdir -p $TMP_BOOTLOADER_DIR if [ $DISTRO_NAME = "fedora" ]; then GRUB_FILE="/boot/efi/EFI/fedora/grubx64.efi" else GRUB_FILE="/usr/lib/grub/x86_64-efi-signed/grubx64.efi.signed" fi if [ $DISTRO_NAME = "fedora" ]; then SHIM_FILE="/boot/efi/EFI/fedora/shim.efi" else SHIM_FILE="/usr/lib/shim/shim.efi.signed" fi #fedora if [ $DISTRO_NAME = "fedora" ]; then SYSLINUX_FILE="/usr/share/syslinux/isolinux.bin" LDLINUX_FILE="/usr/share/syslinux/ldlinux.c32" #debian elif [ $DISTRO_NAME = "debian" ]; then SYSLINUX_FILE="/usr/lib/ISOLINUX/isolinux.bin" LDLINUX_FILE="/usr/lib/syslinux/modules/bios/ldlinux.c32" #ubuntu else SYSLINUX_FILE="/usr/lib/syslinux/isolinux.bin" LDLINUX_FILE="/usr/lib/syslinux/ldlinux.c32" fi cp $GRUB_FILE $TMP_BOOTLOADER_DIR cp $SHIM_FILE $TMP_BOOTLOADER_DIR cp $SYSLINUX_FILE $TMP_BOOTLOADER_DIR # Starting from SYSLINUX 5.00, the isolinux.bin is dependent # on ldlinux.c32. # http://www.syslinux.org/wiki/index.php/Library_modules if [ -f "$LDLINUX_FILE" ]; then cp $LDLINUX_FILE $TMP_BOOTLOADER_DIR fi diskimage-builder-2.11.0/diskimage_builder/elements/local-config/000077500000000000000000000000001324273057700250025ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/local-config/README.rst000066400000000000000000000004571324273057700264770ustar00rootroot00000000000000============ local-config ============ Copies local user settings such as .ssh/authorized\_keys and $http\_proxy into the image. Environment Variables --------------------- DIB_LOCAL_CONFIG_USERNAME :Required: No :Default: root :Description: Username used when installing .ssh/authorized\_keys. diskimage-builder-2.11.0/diskimage_builder/elements/local-config/element-deps000066400000000000000000000000211324273057700273000ustar00rootroot00000000000000package-installs diskimage-builder-2.11.0/diskimage_builder/elements/local-config/environment.d/000077500000000000000000000000001324273057700275705ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/local-config/environment.d/62-ssh-key000066400000000000000000000001041324273057700313160ustar00rootroot00000000000000export DIB_LOCAL_CONFIG_USERNAME=${DIB_LOCAL_CONFIG_USERNAME:-root} diskimage-builder-2.11.0/diskimage_builder/elements/local-config/extra-data.d/000077500000000000000000000000001324273057700272565ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/local-config/extra-data.d/62-ssh-key000077500000000000000000000005431324273057700310160ustar00rootroot00000000000000#!/bin/bash # Save user SSH public key if available. # (Obviously not suitable for downloadable images). if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail [ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set" if [ -e ~/.ssh/authorized_keys ]; then cat ~/.ssh/authorized_keys > $TMP_HOOKS_PATH/ssh-authorized-keys fi diskimage-builder-2.11.0/diskimage_builder/elements/local-config/install.d/000077500000000000000000000000001324273057700266725ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/local-config/install.d/62-ssh-key000077500000000000000000000011201324273057700304220ustar00rootroot00000000000000#!/bin/bash # Save user SSH public key if available. # XXX: Obviously not suitable for downloadable images. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail if [ "$DIB_LOCAL_CONFIG_USERNAME" == "root" ]; then DIR_NAME=${DIB_LOCAL_CONFIG_USERNAME} else DIR_NAME=home/${DIB_LOCAL_CONFIG_USERNAME} fi if [ -e "/tmp/in_target.d/ssh-authorized-keys" ]; then mkdir -p /${DIR_NAME}/.ssh cat /tmp/in_target.d/ssh-authorized-keys >> /${DIR_NAME}/.ssh/authorized_keys chmod 0700 /${DIR_NAME}/.ssh chmod 0600 /${DIR_NAME}/.ssh/authorized_keys fi diskimage-builder-2.11.0/diskimage_builder/elements/local-config/pre-install.d/000077500000000000000000000000001324273057700274565ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/local-config/pre-install.d/02-proxy-settings000077500000000000000000000040331324273057700326420ustar00rootroot00000000000000#!/bin/bash # Save the HTTP/[S] and noproxy settings if available. # XXX: Obviously not suitable for downloadable images. if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail have_apt= have_yum= have_zypper= http_proxy=${http_proxy:-""} https_proxy=${https_proxy:-""} if [ -d /etc/apt ] ; then have_apt=1 fi if [ -e /etc/yum.conf ] ; then have_yum=1 fi if [ -d /etc/zypp ] ; then have_zypper=1 fi if [ -n "$http_proxy" ]; then if [ -d ~stack ]; then echo export http_proxy=$http_proxy >> ~stack/.profile fi if [ -n "$have_apt" ] ; then echo "Acquire::http::Proxy \"$http_proxy\";" > /etc/apt/apt.conf.d/02-use-http-proxy fi if [ -n "$have_yum" ] ; then sed -i -e "s,\[main\],[main]\nproxy=$http_proxy," /etc/yum.conf fi if [ -n "$have_zypper" ] ; then sed -i -e "s,^HTTP_PROXY=.*$,HTTP_PROXY=\"$http_proxy\"," /etc/sysconfig/proxy fi fi if [ -n "$https_proxy" ]; then if [ -d ~stack ]; then echo export https_proxy=$https_proxy >> ~stack/.profile fi if [ -n "$have_apt" ] ; then echo "Acquire::https::Proxy \"$https_proxy\";" > /etc/apt/apt.conf.d/02-use-https-proxy fi if [ -n "$have_zypper" ] ; then sed -i -e "s,^HTTPS_PROXY=.*$,HTTPS_PROXY=\"$https_proxy\"," /etc/sysconfig/proxy fi fi no_proxy=${no_proxy:+"$no_proxy,192.0.2.1"} no_proxy=${no_proxy:-"192.0.2.1"} if [ -n "$no_proxy" ]; then if [ -d ~stack ]; then echo export no_proxy=$no_proxy >> ~stack/.profile fi if [ -n "$have_apt" ] ; then for host in $(sed 's/,/ /g' <<<$no_proxy); do echo "Acquire::http::Proxy::$host \"DIRECT\";" >> /etc/apt/apt.conf.d/02-no-proxy done fi if [ -n "$have_zypper" ] ; then sed -i -e "s,^\(NO_PROXY=.*\)\"$,\1\, $no_proxy\"," /etc/sysconfig/proxy fi fi if [ -n "$http_proxy" -o -n "$https_proxy" -o -n "$no_proxy" ]; then if [ -n "$have_zypper" ] ; then sed -i -e "s,^PROXY_ENABLED=.*$,PROXY_ENABLED=\"yes\"," /etc/sysconfig/proxy fi fi diskimage-builder-2.11.0/diskimage_builder/elements/manifests/000077500000000000000000000000001324273057700244365ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/manifests/README.rst000066400000000000000000000017341324273057700261320ustar00rootroot00000000000000========= manifests ========= An framework for saving manifest information generated during the build for later inspection. Manifests are kept in the final image and also copied to the build area post-image creation. Elements that wish to save any form of manifest should depend on this element and can save their data to into the ``DIB_MANIFEST_IMAGE_DIR`` ( which defaults to ``/etc/dib-manifests``). Note this is created in ``extra-data.d`` rather than ``pre-install.d`` to allow the ``source-repositories`` element to make use of it The manifests are copied to ``DIB_MANIFEST_SAVE_DIR``, which defaults to ``${IMAGE_NAME}.d/``, resulting in the manifests being available as ``${IMAGE_NAME}.d/dib-manifests`` by default after the build. Extra status ------------ This element will also add the files ``dib_environment`` and ``dib_arguments`` to the manifest recording the ``diskimage-builder`` specific environment (``DIB_*`` variables) and command-line arguments respectively. diskimage-builder-2.11.0/diskimage_builder/elements/manifests/cleanup.d/000077500000000000000000000000001324273057700263075ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/manifests/cleanup.d/01-copy-manifests-dir000077500000000000000000000032301324273057700321660ustar00rootroot00000000000000#!/bin/bash # # Copyright 2014 Hewlett-Packard Development Company, L.P. # Copyright 2017 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail MANIFEST_IMAGE_PATH=${TMP_MOUNT_PATH}/${DIB_MANIFEST_IMAGE_DIR} # Double check: directory must be created in extra-data.d/20-manifest-dir [ -d ${MANIFEST_IMAGE_PATH} ] || { echo "Error: MANIFEST_IMAGE_PATH [${MANIFEST_IMAGE_PATH}] does not exist"; exit 1; } echo "$DIB_ENV" | sudo dd of=${MANIFEST_IMAGE_PATH}/dib_environment # dib-lint: safe_sudo echo "$DIB_ARGS" | sudo dd of=${MANIFEST_IMAGE_PATH}/dib_arguments # dib-lint: safe_sudo # Save the manifests locally to the save dir mkdir -p ${DIB_MANIFEST_SAVE_DIR} cp --no-preserve=ownership -rv ${MANIFEST_IMAGE_PATH} ${DIB_MANIFEST_SAVE_DIR} # Lock down permissions on the manifest files inside the image to # root. We don't want regular users being able to see what might # contain a password, etc. find ${MANIFEST_IMAGE_PATH} -type f | xargs sudo chown root:root # dib-lint: safe_sudo find ${MANIFEST_IMAGE_PATH} -type f | xargs sudo chmod 600 # dib-lint: safe_sudo diskimage-builder-2.11.0/diskimage_builder/elements/manifests/environment.d/000077500000000000000000000000001324273057700272245ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/manifests/environment.d/14-manifests000066400000000000000000000013601324273057700313620ustar00rootroot00000000000000# Copyright 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # export DIB_MANIFEST_IMAGE_DIR=${DIB_MANIFEST_IMAGE_DIR:-/etc/dib-manifests} export DIB_MANIFEST_SAVE_DIR=${DIB_MANIFEST_SAVE_DIR:-${IMAGE_NAME}.d/} diskimage-builder-2.11.0/diskimage_builder/elements/manifests/extra-data.d/000077500000000000000000000000001324273057700267125ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/manifests/extra-data.d/20-manifest-dir000077500000000000000000000013621324273057700314430ustar00rootroot00000000000000#!/bin/bash # # Copyright 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail sudo mkdir -p $TMP_MOUNT_PATH/${DIB_MANIFEST_IMAGE_DIR} diskimage-builder-2.11.0/diskimage_builder/elements/mellanox/000077500000000000000000000000001324273057700242645ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/mellanox/README.rst000066400000000000000000000000771324273057700257570ustar00rootroot00000000000000======== mellanox ======== Force support for mellanox hardware diskimage-builder-2.11.0/diskimage_builder/elements/mellanox/element-deps000066400000000000000000000000311324273057700265630ustar00rootroot00000000000000package-installs pkg-map diskimage-builder-2.11.0/diskimage_builder/elements/mellanox/init.d/000077500000000000000000000000001324273057700254515ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/mellanox/init.d/01-mellanox000066400000000000000000000002031324273057700274240ustar00rootroot00000000000000# extra load for mellanox modprobe mlx4_en modprobe mlx4_ib modprobe ib_ipoib modprobe mlx5_ib modprobe ib_umad modprobe ib_uverbs diskimage-builder-2.11.0/diskimage_builder/elements/mellanox/install.d/000077500000000000000000000000001324273057700261545ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/mellanox/install.d/65-mellanox000077500000000000000000000010361324273057700301510ustar00rootroot00000000000000#!/bin/bash # Ensure we load the mellanox driver somehow if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail home=$(dirname $0) install -m 0644 -o root -g root $home/mellanox-rules.udev /etc/udev/rules.d/81-mellanox.rules # needed kernel modules; mlx4_en mlx4_ib ib_ipoib mlx5_ib ib_umad ib_uverbs # mlx5_core loaded by mlx5_ib echo "mlx4_en" >>/etc/modules echo "mlx4_ib" >>/etc/modules echo "ib_ipoib" >>/etc/modules echo "mlx5_ib" >>/etc/modules echo "ib_umad" >>/etc/modules echo "ib_uverbs" >>/etc/modules diskimage-builder-2.11.0/diskimage_builder/elements/mellanox/install.d/mellanox-rules.udev000066400000000000000000000005341324273057700320120ustar00rootroot00000000000000ACTION!="add", GOTO="drivers_end" SUBSYSTEM=="net", RUN+="/sbin/modprobe mlx4_en" SUBSYSTEM=="net", RUN+="/sbin/modprobe mlx4_ib" SUBSYSTEM=="net", RUN+="/sbin/modprobe ib_ipoib" SUBSYSTEM=="net", RUN+="/sbin/modprobe mlx5_ib" SUBSYSTEM=="net", RUN+="/sbin/modprobe ib_umad" SUBSYSTEM=="net", RUN+="/sbin/modprobe ib_uverbs" LABEL="drivers_end" diskimage-builder-2.11.0/diskimage_builder/elements/mellanox/package-installs.yaml000066400000000000000000000004341324273057700303730ustar00rootroot00000000000000dkms: ibacm: ibutils: ibverbs-utils: infiniband-diags: libibcm: libibcommon: libibmad: libibumad: libibverbs: libibverbs-runtime: libmlx4: libmlx4-dev: libmlx5: librdmacm: librdmacm-dev: librdmacm-runtime: mstflint: opensm: pciutils: perftest: qperf: rdma: rpm-build: srptools: vlan: diskimage-builder-2.11.0/diskimage_builder/elements/mellanox/pkg-map000066400000000000000000000037551324273057700255550ustar00rootroot00000000000000{ "family": { "debian":{ "dkms": "dkms", "libibverbs": "libibverbs*", "ibacm": "ibacm", "librdmacm": "librdmacm*", "libmlx4": "libmlx4*", "libmlx5": "libmlx5*", "libibcm": "libibcm.*", "libibmad": "libibmad.*", "libibumad": "libibumad*", "libmlx4-dev": "libmlx4-dev", "librdmacm-dev": "librdmacm-dev", "rdma": "rdmacm-utils", "vlan": "vlan", "ibverbs-utils": "ibverbs-utils" }, "redhat": { "libibverbs": "libibverbs.*", "librdmacm": "librdmacm.*", "libmlx4": "libmlx4.*", "libmlx5": "libmlx5.*", "libibcm": "libibcm.*", "libibmad": "libibmad.*", "libibumad": "libibumad.*", "rdma": "rdma", "qperf": "qperf", "pciutils": "pciutils" }, "suse":{ "libibverbs": "libibverbs", "librdmacm": "librdmacm", "libmlx4": "libmlx4", "libmlx5": "libmlx5", "libibcm": "libibcm", "libibmad": "libibmad", "libibumad": "libibumad", "rdma": "rdma", "qperf": "qperf", "rpm-build": "rpm-build", "libibverbs-runtime": "libibverbs-runtime", "librdmacm-runtime": "librdmacm-runtime", "libibcommon": "libibcommon" } }, "default": { "infiniband-diags": "infiniband-diags", "mstflint": "mstflint", "opensm": "opensm", "srptools": "srptools", "libmlx4-dev": "", "librdmacm-dev": "", "ibutils": "ibutils", "perftest": "perftest", "vlan": "", "pciutils": "", "ibverbs-utils": "", "rpm-build": "", "libibverbs-runtime": "", "librdmacm-runtime": "", "ibacm": "", "qperf": "", "dkms": "", "libibcommon": "" } } diskimage-builder-2.11.0/diskimage_builder/elements/mellanox/udev.d/000077500000000000000000000000001324273057700254515ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/mellanox/udev.d/81-mellanox-drivers.rules000066400000000000000000000005341324273057700322500ustar00rootroot00000000000000ACTION!="add", GOTO="drivers_end" SUBSYSTEM=="net", RUN+="/sbin/modprobe mlx4_en" SUBSYSTEM=="net", RUN+="/sbin/modprobe mlx4_ib" SUBSYSTEM=="net", RUN+="/sbin/modprobe ib_ipoib" SUBSYSTEM=="net", RUN+="/sbin/modprobe mlx5_ib" SUBSYSTEM=="net", RUN+="/sbin/modprobe ib_umad" SUBSYSTEM=="net", RUN+="/sbin/modprobe ib_uverbs" LABEL="drivers_end" diskimage-builder-2.11.0/diskimage_builder/elements/modprobe-blacklist/000077500000000000000000000000001324273057700262225ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/modprobe-blacklist/README.rst000066400000000000000000000005261324273057700277140ustar00rootroot00000000000000================== modprobe-blacklist ================== Blacklist specific modules using modprobe.d/blacklist.conf. In order to use set DIB_MODPROBE_BLACKLIST to the name of your module. To disable multiple modules you can set DIB_MODPROBE_BLACKLIST to a list of string separated by spaces. Example: export DIB_MODPROBE_BLACKLIST="igb" diskimage-builder-2.11.0/diskimage_builder/elements/modprobe-blacklist/install.d/000077500000000000000000000000001324273057700301125ustar00rootroot0000000000000080-modprobe-blacklist000077500000000000000000000004231324273057700337620ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/modprobe-blacklist/install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail MODULES_LIST=${DIB_MODPROBE_BLACKLIST:?"Please set DIB_MODPROBE_BLACKLIST."} for mod_name in $MODULES_LIST; do echo "blacklist $mod_name" >> /etc/modprobe.d/blacklist.conf done diskimage-builder-2.11.0/diskimage_builder/elements/no-final-image/000077500000000000000000000000001324273057700252305ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/no-final-image/README.rst000066400000000000000000000010641324273057700267200ustar00rootroot00000000000000============== no-final-image ============== This is a noop element which can be used to indicate to diskimage-builder that it should not bother creating a final image out of the generated filesystem. It is useful in cases where an element handles all of the image building itself, such as ironic-agent or Docker images. In those cases the final image normally generated by diskimage-builder is not the desired output, so there's no reason to spend time creating it. Elements that wish to behave this way should include this element in their element-deps file. diskimage-builder-2.11.0/diskimage_builder/elements/oat-client/000077500000000000000000000000001324273057700245045ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/oat-client/README.rst000066400000000000000000000011341324273057700261720ustar00rootroot00000000000000========== oat-client ========== This element installs oat-client on the image, that's necessary for trusted boot feature in Ironic to work. Intel TXT will measure BIOS, Option Rom and Kernel/Ramdisk during trusted boot, the oat-client will securely fetch the hash values from TPM. .. note:: This element only works on Fedora. Put `fedora-oat.repo` into `/etc/yum.repos.d/`:: export DIB_YUM_REPO_CONF=/etc/yum.repos.d/fedora-oat.repo .. note:: OAT Repo is lack of a GPG signature check on packages, which can be tracked on: https://github.com/OpenAttestation/OpenAttestation/issues/26 diskimage-builder-2.11.0/diskimage_builder/elements/oat-client/element-deps000066400000000000000000000000211324273057700270020ustar00rootroot00000000000000package-installs diskimage-builder-2.11.0/diskimage_builder/elements/oat-client/package-installs.yaml000066400000000000000000000000351324273057700306100ustar00rootroot00000000000000oat-client: oat-commandtool: diskimage-builder-2.11.0/diskimage_builder/elements/oat-client/yum.repos.d/000077500000000000000000000000001324273057700266675ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/oat-client/yum.repos.d/fedora-oat.repo000066400000000000000000000003431324273057700315770ustar00rootroot00000000000000# Place this file in your /etc/yum.repos.d/ directory [oat] name=oat 2.2 packages and dependencies baseurl=http://repos.fedorapeople.org/repos/gwei3/oat/fedora-$releasever/$basearch/ enabled=1 skip_if_unavailable=1 gpgcheck=0 diskimage-builder-2.11.0/diskimage_builder/elements/openssh-server/000077500000000000000000000000001324273057700254305ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/openssh-server/README.rst000066400000000000000000000010311324273057700271120ustar00rootroot00000000000000============== openssh-server ============== This element ensures that openssh server is installed and enabled during boot. Note ---- Most cloud images come with the openssh server service installed and enabled during boot. However, certain cloud images, especially those created by the \*-minimal elements may not have it installed or enabled. In these cases, using this element may be helpful to ensure your image will accessible via SSH. It's usually helpful to combine this element with others such as the `runtime-ssh-host-keys`. diskimage-builder-2.11.0/diskimage_builder/elements/openssh-server/element-deps000066400000000000000000000000671324273057700277400ustar00rootroot00000000000000dib-init-system package-installs runtime-ssh-host-keys diskimage-builder-2.11.0/diskimage_builder/elements/openssh-server/package-installs.yaml000066400000000000000000000000201324273057700315260ustar00rootroot00000000000000openssh-server: diskimage-builder-2.11.0/diskimage_builder/elements/openssh-server/pkg-map000066400000000000000000000002051324273057700267040ustar00rootroot00000000000000{ "family": { "suse": { "openssh-server": "openssh" }, "gentoo": { "openssh-server": "openssh" } } } diskimage-builder-2.11.0/diskimage_builder/elements/openssh-server/post-install.d/000077500000000000000000000000001324273057700303035ustar00rootroot0000000000000080-enable-sshd-service000077500000000000000000000011651324273057700342250ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/openssh-server/post-install.d#!/bin/bash if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail case "$DIB_INIT_SYSTEM" in upstart) # nothing to do exit 0 ;; systemd) if [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then systemctl enable ssh.service else systemctl enable sshd.service fi ;; openrc) rc-update add sshd default ;; sysv) # ssh is enabled by default, nothing to do exit 0 ;; *) echo "Unsupported init system $DIB_INIT_SYSTEM" exit 1 ;; esac diskimage-builder-2.11.0/diskimage_builder/elements/openstack-ci-mirrors/000077500000000000000000000000001324273057700265205ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/openstack-ci-mirrors/README.rst000066400000000000000000000004611324273057700302100ustar00rootroot00000000000000==================== openstack-ci-mirrors ==================== This element contains various settings to setup mirrors for openstack ci gate testing in a generic fashion. It is intended to be used as a dependency of testing elements that run in the gate. It should do nothing outside that environment. diskimage-builder-2.11.0/diskimage_builder/elements/openstack-ci-mirrors/environment.d/000077500000000000000000000000001324273057700313065ustar00rootroot0000000000000011-dib-distribution-mirror.bash000066400000000000000000000030421324273057700370670ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/openstack-ci-mirrors/environment.d# Set DIB_DISTRIBUTION_MIRROR if running in openstack gate if [ -f /etc/ci/mirror_info.sh ]; then # don't spam logs with this source _xtrace=$(set +o | grep xtrace) set +o xtrace source /etc/ci/mirror_info.sh $_xtrace # note 11- is after 10- which is where DISTRO_NAME is set usually if [[ "${DISTRO_NAME}" == "ubuntu" ]]; then export DIB_DISTRIBUTION_MIRROR=$NODEPOOL_UBUNTU_MIRROR export DIB_DEBOOTSTRAP_EXTRA_ARGS+=" --no-check-gpg" elif [[ "${DISTRO_NAME}" == "debian" ]]; then export DIB_DISTRIBUTION_MIRROR=$NODEPOOL_DEBIAN_MIRROR export DIB_DEBOOTSTRAP_EXTRA_ARGS+=" --no-check-gpg" elif [[ "${DISTRO_NAME}" == "fedora" ]]; then export DIB_DISTRIBUTION_MIRROR=$NODEPOOL_FEDORA_MIRROR elif [[ "${DISTRO_NAME}" == "centos" ]]; then export DIB_DISTRIBUTION_MIRROR=$NODEPOOL_CENTOS_MIRROR elif [[ "${DISTRO_NAME}" == "centos7" ]]; then export DIB_DISTRIBUTION_MIRROR=$NODEPOOL_CENTOS_MIRROR elif [[ "${DISTRO_NAME}" == "opensuse" ]]; then export DIB_DISTRIBUTION_MIRROR=$NODEPOOL_OPENSUSE_MIRROR fi fi # This is repo files pre-created for the fedora/centos-minimal jobs in # the gate if [[ -d ${WORKSPACE:-/not/a/path/}/dib-mirror ]]; then if [[ "${DISTRO_NAME}" == "fedora" ]]; then export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${WORKSPACE}/dib-mirror/fedora-minimal/yum.repos.d elif [[ "${DISTRO_NAME}" == "centos" ]]; then export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${WORKSPACE}/dib-mirror/centos-minimal/yum.repos.d fi fi diskimage-builder-2.11.0/diskimage_builder/elements/openstack-ci-mirrors/root.d/000077500000000000000000000000001324273057700277255ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/openstack-ci-mirrors/root.d/10-apt-disable-gpg000077500000000000000000000020461324273057700330330ustar00rootroot00000000000000#!/bin/bash # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # dib-lint: disable=safe_sudo if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # Disable gpg signing for apt openstack ci repos are created by reprepro # and not signed. # This has to be after debootstrap, but before anything else tries to # use the mirror. APT_DIR=${TARGET_ROOT}/etc/apt/apt.conf.d if [ -d ${APT_DIR} ]; then FILE=${APT_DIR}/99unauthenticated sudo dd of=${FILE} < will cause the list of packages in that file to be installed at the beginning of the respective phase. If the package name in the file starts with a "-", then that package will be removed at the end of the install.d phase. Using post-install.d for cleanup ================================ Package removal is done in post-install.d at level 95. If you a running cleanup functions before this, you need to be careful not to clean out any temporary files relied upon by this element. For this reason, generally post-install cleanup functions should occupy the higher levels between 96 and 99. diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/bin/000077500000000000000000000000001324273057700264375ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/bin/package-installs000077500000000000000000000030141324273057700316050ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail SCRIPTNAME=$(basename $0) function show_options () { echo "Usage: $SCRIPTNAME -d " echo echo "Options:" echo " -d -- directory to search for package-installs-* files" exit 1 } TEMP=$(getopt -o hd: -n $SCRIPTNAME -- "$@") if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi eval set -- "$TEMP" WORKDIR= while true ; do case "$1" in -d) WORKDIR=$2; shift 2;; -h) show_options;; --) shift ; break ;; *) echo "Error: unsupported option $1."; exit 1;; esac done if [ -z "$WORKDIR" ]; then show_options fi PACKAGES= for PACKAGEFILE in $(find $WORKDIR -maxdepth 1 -name "package-installs-*" ); do basefile=$(basename $PACKAGEFILE) element_name=${basefile#"package-installs-"} while read pkg; do if [ -z "$pkg" ]; then continue fi # Ignore comments if [ ${pkg:0:1} = "#" ]; then continue fi if [ ${pkg:0:1} = "-" ]; then pkg=${pkg:1} fi if [ -e /usr/share/pkg-map/$element_name ]; then # map the package to its true name set +e map_pkg=$(pkg-map --element $element_name $pkg) if [ $? -eq 0 ]; then pkg=$map_pkg fi set -e fi PACKAGES="$PACKAGES $pkg" done < $PACKAGEFILE done if [ -n "$PACKAGES" ]; then install-packages $PACKAGES fi diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/bin/package-installs-squash000077500000000000000000000120211324273057700331050ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import argparse import collections import functools import json import logging import os import sys import yaml from diskimage_builder import logging_config logger = logging.getLogger(__name__) def get_element_installtype(element_name): default = os.environ.get("DIB_DEFAULT_INSTALLTYPE", "source") return os.environ.get( "DIB_INSTALLTYPE_%s" % element_name.replace('-', '_'), default) def _is_arch_in_list(strlist): """Checks if os.environ['ARCH'] is in comma separated strlist""" strlist = strlist.split(',') map(str.strip, strlist) return os.environ['ARCH'] in strlist def _valid_for_arch(pkg_name, arch, not_arch): """Filter out incorrect ARCH versions""" if arch is None and not_arch is None: # nothing specified; always OK return True if arch and not_arch: print("package-installs configuration error: arch and not_arch " "given for package [%s]" % pkg_name) sys.exit(1) # if we have an arch list, our current arch must be in it # to install. if arch: return _is_arch_in_list(arch) # if we don't have an explicit arch list, we should # install unless we are in the not-arch list. return not _is_arch_in_list(not_arch) def collect_data(data, filename, element_name): try: objs = json.load(open(filename)) except ValueError: objs = yaml.safe_load(open(filename)) for pkg_name, params in objs.items(): if not params: params = {} phase = params.get('phase', 'install.d') install = "install" if 'uninstall' in params: install = "uninstall" # Filter out incorrect installtypes installtype = params.get('installtype', None) elem_installtype = get_element_installtype(element_name) valid_installtype = (installtype is None or installtype == elem_installtype) valid_arch = _valid_for_arch(pkg_name, params.get('arch', None), params.get('not-arch', None)) dib_py_version = str(params.get('dib_python_version', '')) dib_py_version_env = os.environ.get('DIB_PYTHON_VERSION', '') valid_dib_python_version = (dib_py_version == '' or dib_py_version == dib_py_version_env) if valid_installtype and valid_arch and valid_dib_python_version: data[phase][install].append((pkg_name, element_name)) return data def main(): parser = argparse.ArgumentParser( description="Produce a single packages-installs file from all of" " the available package-installs files") parser.add_argument('--elements', required=True, help="Which elements to squash") parser.add_argument('--path', required=True, help="Elements path to search for elements") parser.add_argument('outfile', help="Location of the output file") args = parser.parse_args() logging_config.setup() # Replicate the logic of finding the first element, because we can't # operate on the post-copied hooks dir, since we lose element context element_dirs = list() for element_name in args.elements.split(): for elements_dir in args.path.split(':'): potential_path = os.path.join(elements_dir, element_name) if os.path.exists(potential_path): element_dirs.append((elements_dir, element_name)) logger.debug("element_dirs -> %s" % element_dirs) # Collect the merge of all of the existing install files in the elements # that are the first on the ELEMENT_PATH final_dict = collections.defaultdict( functools.partial(collections.defaultdict, list)) for (elements_dir, element_name) in element_dirs: for file_type in ('json', 'yaml'): target_file = os.path.join( elements_dir, element_name, "package-installs.%s" % file_type) if not os.path.exists(target_file): continue logger.info("Squashing install file: %s" % target_file) final_dict = collect_data(final_dict, target_file, element_name) logger.debug("final_dict -> %s" % final_dict) # Write the resulting file with open(args.outfile, 'w') as outfile: json.dump( final_dict, outfile, indent=True, separators=(',', ': '), sort_keys=False) if __name__ == '__main__': main() diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/bin/package-installs-v2000077500000000000000000000113271324273057700321400ustar00rootroot00000000000000#!/usr/local/bin/dib-python # Copyright 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from __future__ import print_function import argparse import json import os import subprocess import sys from collections import defaultdict # run a command, return output # if follow is set, output will be echoed to stdout def process_output(cmdline, follow=False): proc = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) if follow: print("Running command: %s" % cmdline) out = "" with proc.stdout: for line in iter(proc.stdout.readline, ''): out += line print("> %s" % line, end="") proc.wait() print("returncode: %d" % proc.returncode) else: out = proc.communicate()[0] if proc.returncode: e = subprocess.CalledProcessError(proc.returncode, cmdline) e.output = out raise e return out def main(): dbg_level = int(os.getenv('DIB_DEBUG_TRACE', '0')) parser = argparse.ArgumentParser( description="Install or uninstall packages for a specific phase based" " on package-installs files.") parser.add_argument('--phase', required=True, help="Install phase to filter on. Valid options are" " 'install.d' or pre-install.d") parser.add_argument('--uninstall', action="store_true", help="Only show packages to uninstall. By default only" " packages to install are shown") parser.add_argument('-n', '--noop', action="store_true", help="Don't actually install, just print the command") parser.add_argument('infile', help="File to process") args = parser.parse_args() packages = json.load(open(args.infile)) if args.uninstall: install = "uninstall" else: install = "install" pkgs = list() if args.phase in packages and install in packages[args.phase]: install_packages = packages[args.phase][install] else: print("Nothing to %s" % install) sys.exit(0) # sort the list by element, this way we only do one pkg-map call # per element by_element = defaultdict(list) for (pkg, element) in install_packages: by_element[element].append(pkg) for element, packages in by_element.items(): print("Map %s for %s: %s" % (install, element, ', '.join(packages))) # Only trace pkg-map for higher levels of debugging. Note # that pkg-map debug output comes out on stderr, which is # captured into the output by process_output. We filter by # "prefix" so we don't think the debug lines are packages! pkg_map_args = ['pkg-map', '--prefix', '-', '--missing-ok', '--element', element] if dbg_level > 1: pkg_map_args.append('--debug') pkg_map_args.extend(packages) try: follow = True if dbg_level > 1 else False map_output = process_output(pkg_map_args, follow=follow) map_output = map_output.strip().split('\n') map_output = [m[1:] for m in map_output if m.startswith('-')] pkgs.extend(map_output) except subprocess.CalledProcessError as e: if e.returncode == 1: if args.noop: pkgs.append(pkg) continue else: print("pkg-map failed") sys.exit(1) elif e.returncode == 2: pkgs.append(pkg) continue install_args = ["install-packages"] if args.uninstall: install_args.append("-e") install_args.extend(list(set(pkgs))) if args.noop: print(" ".join(install_args)) else: try: process_output(install_args, follow=True) except subprocess.CalledProcessError as e: print("install-packages failed with returncode %d" % e.returncode) sys.exit(1) if __name__ == '__main__': main() # Tell emacs to use python-mode # Local variables: # mode: python # End: diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/bin/package-uninstalls000077500000000000000000000026111324273057700321520ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail SCRIPTNAME=$(basename $0) function show_options () { echo "Usage: $SCRIPTNAME -d " echo echo "Options:" echo " -d -- directory to search for package-uninstalls-* files" exit 1 } TEMP=$(getopt -o hd: -n $SCRIPTNAME -- "$@") if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi eval set -- "$TEMP" WORKDIR= while true ; do case "$1" in -d) WORKDIR=$2; shift 2;; -h) show_options;; --) shift ; break ;; *) echo "Error: unsupported option $1."; exit 1;; esac done if [ -z "$WORKDIR" ]; then show_options fi PACKAGES= for PACKAGEFILE in $(find $WORKDIR -maxdepth 1 -name "package-installs-*" ); do basefile=$(basename $PACKAGEFILE) element_name=${basefile#"package-installs-"} while read pkg; do if [ -z "$pkg" ]; then continue fi # Ignore comments if [ ${pkg:0:1} = "#" ]; then continue fi if [ ! ${pkg:0:1} = "-" ]; then continue fi if [ -e /usr/share/pkg-map/$element_name ]; then # map the package to its true name pkg=$(pkg-map --element $element_name $pkg) fi pkg=${pkg:1} PACKAGES="$PACKAGES $pkg" done < $PACKAGEFILE done install-packages -e $PACKAGES diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/element-deps000066400000000000000000000000371324273057700301740ustar00rootroot00000000000000dib-python install-bin pkg-map diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/extra-data.d/000077500000000000000000000000001324273057700301435ustar00rootroot0000000000000099-squash-package-install000077500000000000000000000003761324273057700346200ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/extra-data.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail sudo -E $DIB_PYTHON_EXEC $(dirname $0)/../bin/package-installs-squash --elements="$IMAGE_ELEMENT" --path=$ELEMENTS_PATH $TMP_MOUNT_PATH/tmp/package-installs.json diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/install.d/000077500000000000000000000000001324273057700275575ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/install.d/01-package-installs000077500000000000000000000003001324273057700331360ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail package-installs -d $(dirname $0) package-installs-v2 --phase install.d /tmp/package-installs.json diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/install.d/99-package-uninstalls000077500000000000000000000003161324273057700335310ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail package-uninstalls -d $(dirname $0) package-installs-v2 --phase install.d --uninstall /tmp/package-installs.json diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/post-install.d/000077500000000000000000000000001324273057700305425ustar00rootroot0000000000000000-package-installs000077500000000000000000000003051324273057700340460ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/post-install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail package-installs -d $(dirname $0) package-installs-v2 --phase post-install.d /tmp/package-installs.json 95-package-uninstalls000077500000000000000000000003231324273057700344270ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/post-install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail package-uninstalls -d $(dirname $0) package-installs-v2 --phase post-install.d --uninstall /tmp/package-installs.json diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/pre-install.d/000077500000000000000000000000001324273057700303435ustar00rootroot0000000000000002-package-installs000077500000000000000000000003041324273057700336500ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/pre-install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail package-installs -d $(dirname $0) package-installs-v2 --phase pre-install.d /tmp/package-installs.json 99-package-uninstalls000077500000000000000000000003221324273057700342330ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/package-installs/pre-install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail package-uninstalls -d $(dirname $0) package-installs-v2 --phase pre-install.d --uninstall /tmp/package-installs.json diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/000077500000000000000000000000001324273057700261725ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/README.rst000066400000000000000000000052341324273057700276650ustar00rootroot00000000000000================== pip-and-virtualenv ================== This element installs pip and virtualenv in the image. .. note:: This element setups and Python 2 and Python 3 environment. This means it will bring in python2 packages, so isn't appropriate if you want a python3 only environment. Package install =============== If the package installtype is used then these programs are installed from distribution packages. In this case, ``pip`` and ``virtualenv`` will be installed *only* for the python version identified by ``dib-python`` (i.e. the default python for the platform). Distribution packages have worked out name-spacing such that only python2 or python3 owns common scripts like ``/usr/bin/pip`` (on most platforms, ``pip`` refers to python2 pip, and ``pip3`` refers to python3 pip, although some may choose the reverse). To install pip and virtualenv from package:: export DIB_INSTALLTYPE_pip_and_virtualenv=package Source install ============== Source install is the default. If the source installtype is used, ``pip`` and ``virtualenv`` are installed from the latest upstream releases. Source installs from these tools are not name-spaced. It is inconsistent across platforms if the first or last install gets to own common scripts like ``/usr/bin/pip`` and ``virtualenv``. To avoid inconsistency, we firstly install the packaged python 2 **and** 3 versions of ``pip`` and ``virtualenv``. This prevents a later install of these distribution packages conflicting with the source install. We then overwrite ``pip`` and ``virtualenv`` via ``get-pip.py`` and ``pip`` respectively. The system will be left in the following state: * ``/usr/bin/pip`` : python2 pip * ``/usr/bin/pip2`` : python2 pip (same as prior) * ``/usr/bin/pip3`` : python3 pip * ``/usr/bin/virtualenv`` : python2 virtualenv (note python3 ``virtualenv`` script is *not* installed, see below) Source install is supported on limited platforms. See the code, but this includes Ubuntu and RedHat platforms. Using the tools =============== Due to the essentially unsolvable problem of "who owns the script", it is recommended to *not* call ``pip`` or ``virtualenv`` directly. You can directly call them with the ``-m`` argument to the python interpreter you wish to install with. For example, to create a python3 environment do:: # python3 -m virtualenv myenv # myenv/bin/pip install mytool To install a python2 tool from pip:: # python2 -m pip install mytool In this way, you can always know which interpreter is being used (and affected by) the call. Ordering ======== Any element that uses these commands must be designated as 05-* or higher to ensure that they are first installed. diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/element-deps000066400000000000000000000000761324273057700305020ustar00rootroot00000000000000dib-python install-types package-installs source-repositories diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/install.d/000077500000000000000000000000001324273057700300625ustar00rootroot00000000000000pip-and-virtualenv-source-install/000077500000000000000000000000001324273057700364725ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/install.d04-install-pip000077500000000000000000000117151324273057700411020ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/install.d/pip-and-virtualenv-source-install#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail if [[ $DISTRO_NAME =~ (opensuse|fedora|centos|centos7|rhel7) ]]; then _do_py3=0 packages="python-virtualenv python-pip python-setuptools" if [[ $DISTRO_NAME =~ (fedora) ]]; then _do_py3=1 packages+=" python3-virtualenv python3-pip python3-setuptools" fi # force things to happen so our assumptions hold pip_args="-U --force-reinstall" # GENERAL WARNING : mixing packaged python libraries with # pip-installed versions always creates issues. Upstream # openstack-infra uses this a lot (especially devstack) but be # warned: here be dragons :) # Firstly we want to install the system packages. Otherwise later # on somebody does a "yum install python-virtualenv" and goes and # overwrites the pip installed version with the packaged version, # leading to all sorts of weird version issues. if [[ $DISTRO_NAME = opensuse ]]; then zypper -n install $packages else ${YUM:-yum} install -y $packages fi # install the latest python2 pip; this overwrites packaged pip python /tmp/get-pip.py ${pip_args} # pip and setuptools are closely related; we want to ensure the # latest for sanity. Because distro packages don't include enough # info in the egg for pip to be certain it has fully uninstalled # the old package, for safety we clear it out by hand (this seems # to have been a problem with very old to new updates, # e.g. centos6 to current-era, but less so for smaller jumps). # There is a bit of chicken-and-egg problem with pip in that it # requires setuptools for some operations, such as wheel creation. # But just installing setuptools shouldn't require setuptools # itself, so we are safe for this small section. rm -rf /usr/lib/python2.7/site-packages/setuptools* pip install ${pip_args} setuptools if [[ $_do_py3 -eq 1 ]]; then # Repeat above for python3 # You would think that installing python3 bits first, then # python2 would work -- alas get-pip.py doesn't seem to leave # python3 alone: # https://github.com/pypa/pip/issues/4435 python3 /tmp/get-pip.py ${pip_args} rm -rf /usr/lib/python3.?/site-packages/setuptools* pip3 install ${pip_args} setuptools # reclaim /usr/bin/pip back to pip2 ln -sf /usr/bin/pip2 /usr/bin/pip fi # now install latest virtualenv. it vendors stuff it needs so # doesn't have issues with other system packages. # python[2|3]-virtualenv package has installed versioned scripts # (/usr/bin/virtualenv-[2|3]) but upstream does not! (see [2]). # For consistency, clear them out and then reinstall so we're just # left with python2's version # [2] http://pkgs.fedoraproject.org/cgit/rpms/python-virtualenv.git/tree/python-virtualenv.spec#n116) rm /usr/bin/virtualenv* if [[ $_do_py3 -eq 1 ]]; then pip3 install ${pip_args} virtualenv fi pip install ${pip_args} virtualenv # at this point, we should have the latest # pip/setuptools/virtualenv packages for python2 & 3, and # "/usr/bin/pip" and "/usr/bin/virtualenv" should be python2 # versions. if [[ $DISTRO_NAME = opensuse ]]; then for pkg in virtualenv pip setuptools; do cat - >> /etc/zypp/locks <> ${conf} fi elif [[ $DISTRO_NAME = gentoo ]]; then packages="dev-python/pip dev-python/virtualenv" emerge -U $packages else # pre-install packages so dependencies are there. We will # overwrite with latest below. packages="python-pip python3-pip python-virtualenv" # unfortunately older ubuntu (trusty) doesn't have a # python3-virtualenv package -- it seems it wasn't ready at the # time and you had to use "python -m venv". Since then virtualenv # has gained 3.4 support so the pip install below will work if [[ ${DIB_PYTHON_VERSION} == 3 ]]; then packages=$(echo "$packages" | sed s/python-virtualenv/python3-virtualenv/) fi apt-get -y install $packages # force things to happen so our assumptions hold pip_args="-U --force-reinstall" # These install into /usr/local/bin so override any packages, even # if installed later. python3 /tmp/get-pip.py $pip_args python2 /tmp/get-pip.py $pip_args pip3 install $pip_args virtualenv pip install $pip_args virtualenv fi diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/package-installs.yaml000066400000000000000000000016631324273057700323060ustar00rootroot00000000000000# The python-xml and python3 openSUSE packages contain core python modules # such as the XML one but the python devel packages do not pull them. As # such we need to install them during the pre-install.d phase so they are # available as early as possible. See # - https://bugzilla.suse.com/show_bug.cgi?id=1046990 # - https://review.openstack.org/#/c/478648/ # - https://review.openstack.org/#/c/478650/ # for discussions about this issue. python-xml: phase: pre-install.d dib_python_version: 2 python3: phase: pre-install.d installtype: package dib_python_version: 3 python-pip: installtype: package dib_python_version: 2 python3-pip: installtype: package dib_python_version: 3 python-virtualenv: installtype: package dib_python_version: 2 python3-virtualenv: installtype: package dib_python_version: 3 python-dev: installtype: source dib_python_version: 2 python3-dev: installtype: source dib_python_version: 3 diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/pkg-map000066400000000000000000000011511324273057700274470ustar00rootroot00000000000000{ "family": { "gentoo": { "python-pip": "dev-python/pip", "python3-pip": "dev-python/pip", "python-virtualenv": "dev-python/virtualenv", "python3-virtualenv": "dev-python/virtualenv", "python-dev": "dev-lang/python", "python3-dev": "dev-lang/python" }, "suse": { "python-xml": "python-xml", "python-dev": "python-devel", "python3-dev": "python3-devel", "python3": "python3" }, "redhat": { "python-dev": "python2-devel", "python3-dev": "python3-devel" } }, "default": { "python-xml": "", "python3": "" } } source-repository-pip-and-virtualenv000066400000000000000000000001151324273057700352730ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenvpip-and-virtualenv file /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/test-elements/000077500000000000000000000000001324273057700307635ustar00rootroot00000000000000source-install-fedora/000077500000000000000000000000001324273057700351065ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/test-elementselement-deps000066400000000000000000000000441324273057700374110ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/test-elements/source-install-fedorafedora-minimal openstack-ci-mirrors environment.d/000077500000000000000000000000001324273057700376745ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/test-elements/source-install-fedora10-source-installtype-pip-and-virtualenv000066400000000000000000000000611324273057700474230ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/test-elements/source-install-fedora/environment.dexport DIB_INSTALLTYPE_pip_and_virtualenv=source source-install-ubuntu/000077500000000000000000000000001324273057700351705ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/test-elementselement-deps000066400000000000000000000000441324273057700374730ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/test-elements/source-install-ubuntuopenstack-ci-mirrors ubuntu-minimal environment.d/000077500000000000000000000000001324273057700377565ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/test-elements/source-install-ubuntu10-source-installtype-pip-and-virtualenv000066400000000000000000000000611324273057700475050ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pip-and-virtualenv/test-elements/source-install-ubuntu/environment.dexport DIB_INSTALLTYPE_pip_and_virtualenv=source diskimage-builder-2.11.0/diskimage_builder/elements/pip-cache/000077500000000000000000000000001324273057700242765ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pip-cache/README.rst000066400000000000000000000013701324273057700257660ustar00rootroot00000000000000========= pip-cache ========= # Use a cache for pip Using a download cache speeds up image builds. Including this element in an image build causes $HOME/.cache/image-create/pip to be bind mounted as /tmp/pip inside the image build chroot. The $PIP_DOWNLOAD_CACHE environment variable is then defined as /tmp/pip, which causes pip to cache all downloads to the defined location. Note that pip and its use of $PIP_DOWNLOAD_CACHE is not concurrency safe. Running multiple instances of diskimage-builder concurrently can cause issues. Therefore, it is advised to only have one instance of diskimage-builder that includes the pip-cache element running at a time. The pip concurrency issue is being tracked upstream at https://github.com/pypa/pip/issues/1141 diskimage-builder-2.11.0/diskimage_builder/elements/pip-cache/environment.d/000077500000000000000000000000001324273057700270645ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pip-cache/environment.d/10-pip-cache000066400000000000000000000000431324273057700310530ustar00rootroot00000000000000export PIP_DOWNLOAD_CACHE=/tmp/pip diskimage-builder-2.11.0/diskimage_builder/elements/pip-cache/root.d/000077500000000000000000000000001324273057700255035ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pip-cache/root.d/01-pip-cache000077500000000000000000000003701324273057700275000ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail PIP_CACHE_DIR=$DIB_IMAGE_CACHE/pip mkdir -p $PIP_CACHE_DIR sudo mkdir -p $TMP_MOUNT_PATH/tmp/pip sudo mount --bind $PIP_CACHE_DIR $TMP_MOUNT_PATH/tmp/pip diskimage-builder-2.11.0/diskimage_builder/elements/pkg-map/000077500000000000000000000000001324273057700240015ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pkg-map/README.rst000066400000000000000000000051451324273057700254750ustar00rootroot00000000000000======= pkg-map ======= Map package names to distro specific packages. Provides the following: * bin/pkg-map:: usage: pkg-map [-h] [--element ELEMENT] [--distro DISTRO] Translate package name to distro specific name. optional arguments: -h, --help show this help message and exit --element ELEMENT The element (namespace) to use for translation. --distro DISTRO The distro name to use for translation. Defaults to DISTRO_NAME --release RELEASE The release to use for translation. Defaults to DIB_RELEASE * Any element may create its own pkg-map JSON config file using the one of 4 sections for the release/distro/family/ and or default. The family is set automatically within pkg-map based on the supplied distro name. Families include: + redhat: includes centos, fedora, and rhel distros + debian: includes debian and ubuntu distros + suse: includes the opensuse distro The release is a specification of distro; i.e. the distro and release must mach for a translation. The most specific section takes priority. An empty package list can be provided. Example for Nova and Glance (NOTE: using fictitious package names for Fedora and package mapping for suse family to provide a good example!) Example format:: { "release": { "fedora": { "23": { "nova_package": "foo" "bar" } } }, "distro": { "fedora": { "nova_package": "openstack-compute", "glance_package": "openstack-image" } }, "family": { "redhat": { "nova_package": "openstack-nova", "glance_package": "openstack-glance" }, "suse": { "nova_package": "" } }, "default": { "nova_package": "nova", "glance_package": "glance" } } Example commands using this format: pkg-map --element nova-compute --distro fedora nova_package Returns: openstack-compute pkg-map --element nova-compute --distro rhel nova_package Returns: openstack-nova pkg-map --element nova-compute --distro ubuntu nova_package Returns: nova pkg-map --element nova-compute --distro opensuse nova_package Returns: * This output can be used to filter what other tools actually install (install-packages can be modified to use this for example) * Individual pkg-map files live within each element. For example if you are created an Apache element your pkg-map JSON file should be created at elements/apache/pkg-map. diskimage-builder-2.11.0/diskimage_builder/elements/pkg-map/bin/000077500000000000000000000000001324273057700245515ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pkg-map/bin/pkg-map000077500000000000000000000150571324273057700260430ustar00rootroot00000000000000#!/usr/local/bin/dib-python # Copyright 2014 Red Hat Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import argparse import json import logging import os import sys log = logging.getLogger() def os_family(distro): family = None if distro in ['fedora', 'rhel', 'rhel7', 'centos', 'centos7']: family = 'redhat' elif distro in ['debian', 'ubuntu']: family = 'debian' elif distro == 'opensuse': family = 'suse' elif distro == 'gentoo': family = 'gentoo' return family def main(): parser = argparse.ArgumentParser( description="Translate package name to distro specific name." " Exits with 1 if error is encountered, 2 if no pkg-map" " file is found. Otherwise exits with 0.") parser.add_argument('--element', default='', help='The element (namespace) to use for translation.') parser.add_argument('--pkg-map', default='', help='Path to specific pkg-map file. ' '(Useful for testing)') parser.add_argument('--distro', default=os.environ.get('DISTRO_NAME'), help='The distro name to use for translation.' ' Defaults to DISTRO_NAME') parser.add_argument('--release', default=os.environ.get('DIB_RELEASE'), help='A more specfic name for distribution release') parser.add_argument('--missing-ok', action="store_true", help='Do not consider missing mappings an error.' ' Causes packages where no mapping is set to be' ' printed.') # This tool has traditionally output status and debug messages on # stderr. The problem is if a caller has stderr > stdout then # actual output gets messed in with the logs. This allows callers # to disambiguate actual output by specifying a unique prefix. parser.add_argument('--prefix', default='', help='Output mapped packages with this prefix') parser.add_argument('--debug', dest='debug', action="store_true", help="Enable debugging output") args, extra = parser.parse_known_args() # Logs have traditionally gone to stderr with this tool. Maintain # compatibility level = logging.DEBUG if args.debug else logging.INFO logging.basicConfig(stream=sys.stderr, level=level) if not args.element and not args.pkg_map: log.error('Please specify an --element argument.') sys.exit(1) if args.element and args.pkg_map: log.error('Specify either --element or --pkg-map') sys.exit(1) if not args.distro: log.error('Please specify a --distro argument or set DISTRO_NAME.') sys.exit(1) if args.pkg_map: # specifying the pkg-map by hand is just for manual testing element = "<%s>" % args.pkg_map map_file = args.pkg_map else: element = args.element map_file = '/usr/share/pkg-map/%s' % element log.info("Mapping for %s : %s" % (element, ' '.join(extra))) if not os.path.exists(map_file): if args.missing_ok: log.info("No package map for %s, done" % element) for name in extra: print('%s%s' % (args.prefix, name)) sys.exit(0) else: log.error('Required pkg-map for %s element does not exist.' % args.element) sys.exit(2) with open(map_file) as fd: try: package_names = json.loads(fd.read()) # log.debug(pprint.pformat(package_names)) except ValueError: log.error('Unable to parse %s' % map_file) raise # Parse mapping data in this form using release/distro/family/default # Most specific takes priority; order is # - release # - distro # - family # - default # An empty package list can be provided. # # Example for Nova and Glance (using fictitious name for Fedora and SUSE # and package mapping for SUSE family) # { # "release": { # "fedora" : { # "23" : { # "nova_package": "openstack-compute-foo" # } # } # } # "distro": { # "fedora": { # "nova_package": "openstack-compute", # "glance_package": "openstack-image" # } # }, # "family": { # "redhat": { # "nova_package": "openstack-nova", # "glance_package": "openstack-glance" # }, # "suse": { # "nova_package": "" # } # }, # "default": { # "nova_package": "nova", # "glance_package": "glance" # } # } name_map = package_names.get('default', {}) if 'family' in package_names: family_map = package_names['family'].get(os_family(args.distro)) if family_map: name_map.update(family_map) if 'distro' in package_names: distro_map = package_names['distro'].get(args.distro) if distro_map: name_map.update(distro_map) if 'release' in package_names: try: # release is a sub-concept of distro release_map = package_names['release'][args.distro][args.release] name_map.update(release_map) except KeyError: pass # log.debug(pprint.pformat(name_map)) for name in extra: pkg_name = name_map.get(name) if pkg_name: log.debug("map %s -> %s" % (name, pkg_name)) print('%s%s' % (args.prefix, pkg_name)) elif name in name_map: log.debug("map %s -> " % (name)) continue else: if args.missing_ok: log.debug("pass -> %s" % (name)) print('%s%s' % (args.prefix, name)) else: log.error("%s has no valid mapping for package %s" % (element, name)) sys.exit(1) sys.exit(0) if __name__ == '__main__': main() # Tell emacs to use python-mode # Local variables: # mode: python # End: diskimage-builder-2.11.0/diskimage_builder/elements/pkg-map/element-deps000066400000000000000000000000131324273057700263000ustar00rootroot00000000000000dib-python diskimage-builder-2.11.0/diskimage_builder/elements/pkg-map/extra-data.d/000077500000000000000000000000001324273057700262555ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pkg-map/extra-data.d/10-create-pkg-map-dir000077500000000000000000000007031324273057700317720ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail sudo mkdir -p $TMP_MOUNT_PATH/usr/share/pkg-map/ eval declare -A image_elements=($(get_image_element_array)) for i in "${!image_elements[@]}"; do element=$i element_dir=${image_elements[$i]} pkg_map="${element_dir}/pkg-map" if [ -f "${pkg_map}" ]; then sudo cp "${pkg_map}" "$TMP_MOUNT_PATH/usr/share/pkg-map/$element" fi done diskimage-builder-2.11.0/diskimage_builder/elements/posix/000077500000000000000000000000001324273057700236075ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/posix/README.rst000066400000000000000000000007401324273057700252770ustar00rootroot00000000000000posix ===== * This element installs packages to ensure that the resulting image has binaries necessary to meet the requirements of POSIX, laid out in the following URL: + http://pubs.opengroup.org/onlinepubs/9699919799/idx/utilities.html * This has been tested to work on Ubuntu, Debian, and CentOS, although should work on Red Hat Enterprise Linux. * To add support for other distros please consult the URL for binaries, then add the providing packages to pkg-map. diskimage-builder-2.11.0/diskimage_builder/elements/posix/element-deps000066400000000000000000000000171324273057700261120ustar00rootroot00000000000000openssh-server diskimage-builder-2.11.0/diskimage_builder/elements/posix/package-installs.yaml000066400000000000000000000024601324273057700277170ustar00rootroot00000000000000"admin": "alias": "ar": "asa": "at": "awk": "basename": "batch": "bc": "bg": "c99": "cal": "cat": "cd": "cflow": "chgrp": "chmod": "chown": "cksum": "cmp": "comm": "command": "compress": "cp": "crontab": "csplit": "ctags": "cut": "cxref": "date": "dd": "delta": "df": "diff": "dirname": "du": "echo": "ed": "env": "ex": "expand": "expr": "false": "fc": "fg": "file": "find": "fold": "fort77": "fuser": "gencat": "get": "getconf": "getopts": "grep": "hash": "head": "iconv": "id": "ipcrm": "ipcs": "jobs": "join": "kill": "lex": "link": "ln": "locale": "localedef": "logger": "logname": "lp": "ls": "m4": "mailx": "make": "man": "mesg": "mkdir": "mkfifo": "more": "mv": "newgrp": "nice": "nl": "nm": "nohup": "od": "paste": "patch": "pathchk": "pax": "pr": "printf": "prs": "ps": "pwd": "qalter": "qdel": "qhold": "qmove": "qmsg": "qrerun": "qrls": "qselect": "qsig": "qstat": "qsub": "read": "renice": "rm": "rmdel": "rmdir": "sact": "sccs": "sed": "sh": "sleep": "sort": "split": "strings": "strip": "stty": "tabs": "tail": "talk": "tee": "test": "time": "touch": "tput": "tr": "true": "tsort": "tty": "type": "ulimit": "umask": "unalias": "uname": "uncompress": "unexpand": "unget": "uniq": "unlink": "uucp": "uudecode": "uuencode": "uustat": "uux": "val": "vi": "wait": "wc": "what": "who": "write": "xargs": "yacc": "zcat": diskimage-builder-2.11.0/diskimage_builder/elements/posix/pkg-map000066400000000000000000000172711324273057700250760ustar00rootroot00000000000000{ "family":{ "debian":{ "admin": "", "alias": "", "ar": "binutils", "asa": "", "at": "at", "awk": "gawk", "basename": "coreutils", "batch": "at", "bc": "bc", "bg": "", "c99": "gcc", "cal": "bsdmainutils", "cat": "coreutils", "cd": "", "cflow": "cflow", "chgrp": "coreutils", "chmod": "coreutils", "chown": "coreutils", "cksum": "coreutils", "cmp": "diffutils", "comm": "coreutils", "command": "", "compress": "ncompress", "cp": "coreutils", "crontab": "cron", "csplit": "coreutils", "ctags": "exuberant-ctags", "cut": "coreutils", "cxref": "cxref", "date": "coreutils", "dd": "coreutils", "delta": "", "df": "coreutils", "diff": "diffutils", "dirname": "coreutils", "du": "coreutils", "echo": "coreutils", "ed": "ed", "env": "coreutils", "ex": "vim", "expand": "coreutils", "expr": "coreutils", "false": "coreutils", "fc": "", "fg": "", "file": "file", "find": "findutils", "fold": "coreutils", "fort77": "fort77", "fuser": "psmisc", "gencat": "libc-dev-bin", "get": "", "getconf": "libc-bin", "getopts": "", "grep": "grep", "hash": "", "head": "coreutils", "iconv": "libc-bin", "id": "coreutils", "ipcrm": "util-linux", "ipcs": "util-linux", "jobs": "", "join": "coreutils", "kill": "procps", "lex": "flex", "link": "coreutils", "ln": "coreutils", "locale": "libc-bin", "localedef": "libc-bin", "logger": "bsdutils", "logname": "coreutils", "lp": "cups-client", "ls": "coreutils", "m4": "m4", "mailx": "mailutils", "make": "make", "man": "man-db", "mesg": "sysvinit-utils", "mkdir": "coreutils", "mkfifo": "coreutils", "more": "util-linux", "mv": "coreutils", "newgrp": "login", "nice": "coreutils", "nl": "coreutils", "nm": "binutils", "nohup": "coreutils", "od": "coreutils", "paste": "coreutils", "patch": "patch", "pathchk": "coreutils", "pax": "pax", "pr": "coreutils", "printf": "coreutils", "prs": "", "ps": "procps", "pwd": "coreutils", "qalter": "", "qdel": "", "qhold": "", "qmove": "", "qmsg": "", "qrerun": "", "qrls": "", "qselect": "", "qsig": "", "qstat": "", "qsub": "", "read": "", "renice": "bsdutils", "rm": "coreutils", "rmdel": "cssc", "rmdir": "coreutils", "sact": "cssc", "sccs": "cssc", "sed": "sed", "sh": "dash", "sleep": "coreutils", "sort": "coreutils", "split": "coreutils", "strings": "binutils", "strip": "binutils", "stty": "coreutils", "tabs": "ncurses-bin", "tail": "coreutils", "talk": "talk", "tee": "coreutils", "test": "coreutils", "time": "", "touch": "coreutils", "tput": "ncurses-bin", "tr": "coreutils", "true": "coreutils", "tsort": "coreutils", "tty": "coreutils", "type": "", "ulimit": "", "umask": "", "unalias": "", "uname": "coreutils", "uncompress": "gzip", "unexpand": "coreutils", "unget": "", "uniq": "coreutils", "unlink": "coreutils", "uucp": "uucp", "uudecode": "sharutils", "uuencode": "sharutils", "uustat": "uucp", "uux": "uucp", "val": "", "vi": "vim", "wait": "", "wc": "coreutils", "what": "", "who": "coreutils", "write": "bsdmainutils", "xargs": "findutils", "yacc": "bison", "zcat": "gzip" }, "redhat":{ "admin": "", "alias": "bash", "ar": "binutils", "asa": "", "at": "at", "awk": "gawk", "basename": "coreutils", "batch": "at", "bc": "bc", "bg": "", "c99": "gcc", "cal": "util-linux", "cat": "coreutils", "cd": "bash", "cflow": "", "chgrp": "coreutils", "chmod": "coreutils", "chown": "coreutils", "cksum": "coreutils", "cmp": "diffutils", "comm": "coreutils", "command": "bash", "compress": "ncompress", "cp": "coreutils", "crontab": "cronie", "csplit": "coreutils", "ctags": "ctags", "cut": "coreutils", "cxref": "", "date": "coreutils", "dd": "coreutils", "delta": "", "df": "coreutils", "diff": "diffutils", "dirname": "coreutils", "du": "coreutils", "echo": "coreutils", "ed": "ed", "env": "coreutils", "ex": "vim-minimal", "expand": "coreutils", "expr": "coreutils", "false": "coreutils", "fc": "", "fg": "", "file": "file", "find": "findutils", "fold": "coreutils", "fort77": "", "fuser": "psmisc", "gencat": "glibc-common", "get": "", "getconf": "glibc-common", "getopts": "bash", "grep": "grep", "hash": "", "head": "coreutils", "iconv": "glibc-common", "id": "coreutils", "ipcrm": "util-linux", "ipcs": "util-linux", "jobs": "bash", "join": "coreutils", "kill": "util-linux", "lex": "flex", "link": "coreutils", "ln": "coreutils", "locale": "glibc-common", "localedef": "", "logger": "util-linux", "logname": "coreutils", "lp": "cups-client", "ls": "coreutils", "m4": "m4", "mailx": "mailx", "make": "make", "man": "man-db", "mesg": "sysvinit-utils", "mkdir": "coreutils", "mkfifo": "coreutils", "more": "util-linux", "mv": "coreutils", "newgrp": "shadow-utils", "nice": "coreutils", "nl": "coreutils", "nm": "binutils", "nohup": "coreutils", "od": "coreutils", "paste": "coreutils", "patch": "patch", "pathchk": "coreutils", "pax": "pax", "pr": "coreutils", "printf": "coreutils", "prs": "", "ps": "procps-ng", "pwd": "coreutils", "qalter": "", "qdel": "", "qhold": "", "qmove": "", "qmsg": "", "qrerun": "", "qrls": "", "qselect": "", "qsig": "", "qstat": "", "qsub": "", "read": "bash", "renice": "util-linux", "rm": "coreutils", "rmdel": "", "rmdir": "coreutils", "sact": "", "sccs": "", "sed": "sed", "sh": "bash", "sleep": "coreutils", "sort": "coreutils", "split": "coreutils", "strings": "binutils", "strip": "binutils", "stty": "coreutils", "tabs": "ncurses-bin", "tail": "coreutils", "talk": "talk", "tee": "coreutils", "test": "coreutils", "time": "", "touch": "coreutils", "tput": "ncurses", "tr": "coreutils", "true": "coreutils", "tsort": "coreutils", "tty": "coreutils", "type": "", "ulimit": "", "umask": "bash", "unalias": "bash", "uname": "coreutils", "uncompress": "ncompress", "unexpand": "coreutils", "unget": "", "uniq": "coreutils", "unlink": "coreutils", "uucp": "", "uudecode": "sharutils", "uuencode": "sharutils", "uustat": "", "uux": "", "val": "", "vi": "vim-minimal", "wait": "bash", "wc": "coreutils", "what": "", "who": "coreutils", "write": "util-linux", "xargs": "findutils", "yacc": "byacc", "zcat": "gzip" } } } diskimage-builder-2.11.0/diskimage_builder/elements/proliant-tools/000077500000000000000000000000001324273057700254335ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/proliant-tools/README.rst000066400000000000000000000036361324273057700271320ustar00rootroot00000000000000proliant-tools ============== * This element can be used when building ironic-agent ramdisk. It enables ironic-agent ramdisk to do in-band cleaning operations specific to HPE ProLiant hardware. * Works with ubuntu and fedora distributions (on which ironic-agent element is supported). * Currently the following utilities are installed: + `proliantutils`_ - This module registers an ironic-python-agent hardware manager for HPE ProLiant hardware, which implements in-band cleaning steps. The latest version of ``proliantutils`` available is installed. This python module is released with Apache license. + `HPE Smart Storage Administrator (HPE SSA) CLI for Linux 64-bit`_ - This utility is used by ``proliantutils`` library above for doing in-band RAID configuration on HPE ProLiant hardware. Currently installed version is 2.60. Newer version of ``ssacli`` when available, may be installed to the ramdisk by using the environment variable ``DIB_SSACLI_URL``. ``DIB_SSACLI_URL`` should contain the HTTP(S) URL for downloading the RPM package for ``ssacli`` utility. The old environmental variable ``DIB_HPSSACLI_URL``,a HTTP(S) URL for downloading the RPM package for ``hpssacli`` utility, is deprecated. The ``hpssacli`` utility is not supported anymore, use ``ssacli`` instead for the same functionality. Availability of newer versions can be in the Revision History in the above link. This utility is closed source and is released with `HPE End User License Agreement – Enterprise Version`_. .. _`proliantutils`: https://pypi.python.org/pypi/proliantutils .. _`HPE Smart Storage Administrator (HPE SSA) CLI for Linux 64-bit`: http://h20564.www2.hpe.com/hpsc/swd/public/detail?swItemId=MTX_5530b3f5b38b4e0781e6bf9c74 .. _`HPE End User License Agreement – Enterprise Version`: https://downloads.hpe.com/pub/softlib2/software1/doc/p1796552785/v113125/eula-en.html diskimage-builder-2.11.0/diskimage_builder/elements/proliant-tools/element-deps000066400000000000000000000000441324273057700277360ustar00rootroot00000000000000package-installs pip-and-virtualenv diskimage-builder-2.11.0/diskimage_builder/elements/proliant-tools/install.d/000077500000000000000000000000001324273057700273235ustar00rootroot0000000000000065-proliant-tools-install000077500000000000000000000032401324273057700340530ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/proliant-tools/install.d#!/bin/bash # # Copyright 2015 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail # Set the below variable to allow ssacli to be installed from custom URLs. if [[ -n "${DIB_HPSSACLI_URL:=}" ]]; then echo "The environment variable DIB_HPSSACLI_URL is deprecated; use DIB_SSACLI_URL instead." fi DIB_SSACLI_URL=${DIB_SSACLI_URL:-${DIB_HPSSACLI_URL:-https://downloads.hpe.com/pub/softlib2/software1/pubsw-linux/p1857046646/v122802/ssacli-3.10-3.0.x86_64.rpm}} curl -k -o /tmp/ssacli.rpm $DIB_SSACLI_URL if [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then # There is no deb package for ssacli. Install with alien. alien -i /tmp/ssacli.rpm else rpm -iv /tmp/ssacli.rpm fi rm -f /tmp/ssacli.rpm # Install proliantutils python module in the # virtual environment of ironic-python-agent. # This is pre-installed by # ironic-agent/install.d/ironic-agent-source-install/60-ironic-agent-install. IPA_VENV=/usr/share/ironic-python-agent/venv if [[ -d "$IPA_VENV" ]]; then $IPA_VENV/bin/pip install proliantutils else pip install proliantutils fi diskimage-builder-2.11.0/diskimage_builder/elements/proliant-tools/package-installs.yaml000066400000000000000000000000371324273057700315410ustar00rootroot00000000000000alien: gcc: python-dev: unzip: diskimage-builder-2.11.0/diskimage_builder/elements/proliant-tools/pkg-map000066400000000000000000000005441324273057700267150ustar00rootroot00000000000000{ "family": { "debian": { "alien": "alien" }, "redhat": { "python-dev": "python2-devel" }, "suse": { "python-dev": "python-devel" } }, "default": { "alien": "", "gcc": "gcc", "python-dev": "python-dev", "unzip": "unzip" } } diskimage-builder-2.11.0/diskimage_builder/elements/pypi/000077500000000000000000000000001324273057700234265ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pypi/README.rst000066400000000000000000000050321324273057700251150ustar00rootroot00000000000000==== pypi ==== Inject a PyPI mirror ==================== Use a custom PyPI mirror to build images. The default is to bind mount one from ~/.cache/image-create/pypi/mirror into the build environment as mirror URL file:///tmp/pypi. The element temporarily overwrites /root/.pip.conf and .pydistutils.cfg to use it. When online, the official pypi.python.org pypi index is supplied as an extra-url, so uncached dependencies will still be available. When offline, only the mirror is used - be warned that a stale mirror will cause build failures. To disable the pypi.python.org index without using --offline (e.g. when working behind a corporate firewall that prohibits pypi.python.org) set DIB\_NO\_PYPI\_PIP to any non-empty value. To use an arbitrary mirror set DIB\_PYPI\_MIRROR\_URL=http[s]://somevalue/ Additional mirrors can be added by exporting DIB\_PYPI\_MIRROR\_URL\_1=... etc. Only the one mirror can be used by easy-install, but since wheels need to be in the first mirror to be used, the last listed mirror is used as the pydistutils index. NB: The sort order for these variables is a simple string sort - if you have more than 9 additional mirrors, some care will be needed. You can also set the number of retries that occur on failure by setting the DIB\_PIP\_RETRIES environment variable. If setting fallback pip mirrors you typically want to set this to 0 to prevent the need to fail multiple times before falling back. A typical use of this element is thus: export DIB\_PYPI\_MIRROR\_URL=http://site/pypi/Ubuntu-13.10 export DIB\_PYPI\_MIRROR\_URL\_1=http://site/pypi/ export DIB\_PYPI\_MIRROR\_URL\_2=file:///tmp/pypi export DIB\_PIP\_RETRIES=0 [devpi-server](https://git.openstack.org/cgit/openstack-infra/pypi-mirro://pypi.python.org/pypi/devpi-server) can be useful in making a partial PyPI mirror suitable for building images. For instance: * pip install -U devpi * devpi-server quickstart * devpi use http://machinename:3141 * Re-export your variables to point at the new mirror: export DIB\_PYPI\_MIRROR\_URL=http://machinename:3141/ unset DIB\_PYPI\__MIRROR\_URL\_1 unset DIB\_PYPI\__MIRROR\_URL\_2 The next time packages are installed, they'll be cached on the local devpi server; subsequent runs pointed at the same mirror will use the local cache if the upstream can't be contacted. Note that this process only has the server running temporarily; see [Quickstart: Permanent install on server/laptop](http://doc.devpi.net/latest/quickstart-server.html) guide from the devpi developers for more information on a more permanent setup. diskimage-builder-2.11.0/diskimage_builder/elements/pypi/extra-data.d/000077500000000000000000000000001324273057700257025ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pypi/extra-data.d/00-mount-pypi-mirror000077500000000000000000000005511324273057700314770ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail MIRROR_SOURCE=$DIB_IMAGE_CACHE/pypi/mirror/ if [ -d "$MIRROR_SOURCE" ]; then MIRROR_TARGET=$TMP_MOUNT_PATH/tmp/pypi sudo mkdir -p $MIRROR_SOURCE $MIRROR_TARGET # dib-lint: safe_sudo sudo mount --bind $MIRROR_SOURCE $MIRROR_TARGET # dib-lint: safe_sudo fi diskimage-builder-2.11.0/diskimage_builder/elements/pypi/post-install.d/000077500000000000000000000000001324273057700263015ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pypi/post-install.d/00-unconfigure-pypi-mirror000077500000000000000000000004561324273057700332640ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail if [ -e ~/.pip/pip.conf.orig ]; then mv ~/.pip/pip.conf{.orig,} else rm ~/.pip/pip.conf fi if [ -e ~/.pydistutils.cfg.orig ]; then mv ~/.pydistutils.cfg{.orig,} else rm ~/.pydistutils.cfg fi diskimage-builder-2.11.0/diskimage_builder/elements/pypi/pre-install.d/000077500000000000000000000000001324273057700261025ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/pypi/pre-install.d/00-configure-pypi-mirror000077500000000000000000000053021324273057700325150ustar00rootroot00000000000000#!/usr/bin/env python # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from __future__ import print_function import os.path def main(): home = os.path.expanduser("~") backup_configs(home) indices = [] if (os.environ.get('DIB_PYPI_MIRROR_URL') or os.environ.get('PYPI_MIRROR_URL')): candidates = [k for k in os.environ if k.startswith('DIB_PYPI_MIRROR_URL')] back_compat = [k for k in os.environ if k.startswith('PYPI_MIRROR_URL')] if len(back_compat) > 0: print('DEPRECATION WARNING: Please use DIB_PYPI_MIRROR_URL* rather' ' than PYPI_MIRROR_URL*') candidates += back_compat indices = list(map(os.environ.get, sorted(candidates))) else: indices = ['file:///tmp/pypi'] print('WARNING: You are using the pypi element but no pypi mirror is ' 'defined via the PYPI_MIRROR_URL env variable') easy_index = indices[-1] use_pypi_python_org = True if os.environ.get('DIB_OFFLINE'): use_pypi_python_org = False if os.environ.get('DIB_NO_PYPI_PIP'): use_pypi_python_org = False if use_pypi_python_org: indices.append('https://pypi.python.org/simple') retries = os.environ.get('DIB_PIP_RETRIES') with open(home + '/.pip/pip.conf', 'wt') as output: output.write('[global]\n') output.write('log = %s/pip.log\n' % (home,)) output.write('index-url = %s\n' % (indices[0],)) if retries is not None: output.write('retries = %s\n' % retries) for index in indices[1:]: output.write('extra-index-url = %s\n' % (index,)) with open(home + '/.pydistutils.cfg', 'wt') as output: output.write('[easy_install]\n') output.write('index_url = %s\n' % (easy_index,)) def backup_configs(home): pip_dir = home + '/.pip' if not os.path.exists(pip_dir): os.mkdir(pip_dir) if os.path.exists(pip_dir + '/pip.conf'): os.rename(pip_dir + '/pip.conf', pip_dir + '/pip.conf.orig') if os.path.exists(home + '/.pydistutils.cfg'): os.rename(home + '/.pydistutils.cfg', home + '/.pydistutils.cfg.orig') if __name__ == '__main__': main() diskimage-builder-2.11.0/diskimage_builder/elements/python-brickclient/000077500000000000000000000000001324273057700262555ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/python-brickclient/README.rst000066400000000000000000000034371324273057700277530ustar00rootroot00000000000000python-brickclient ================== * This element is aimed for providing cinder local attach/detach functionality. * Currently the feature has a dependency on a known bug ``__, which has been resolved and will be part of the upstream with the next release of ``python-brick-cinderclient-ext``. Note: Current version of ``python-brick-cinderclient-ext`` i.e. 0.2.0 requires and update to be made in Line32 for ``/usr/share/python-brickclient/venv/lib/python2.7/site-packages/brick_cinderclient_ext/__init__.py``: update ``brick-python-cinderclient-ext`` to ``python-brick-cinderclient-ext``. Usage ----- Pass the below shell script to parameter ``user-data`` and set ``config-drive=true`` at the time of provisioning the node via nova-boot to make cinder local attach/detach commands talk to your cloud controller. .. code-block:: bash #!/bin/bash FILE="/etc/bash.bashrc" [ ! -f "$FILE" ] && touch "$FILE" echo 'export OS_AUTH_URL="http://:5000/v2.0"' >> "$FILE" echo 'export OS_PASSWORD="password"' >> "$FILE" echo 'export OS_USERNAME="demo"' >> "$FILE" echo 'export OS_TENANT_NAME="demo"' >> "$FILE" echo 'export OS_PROJECT_NAME="demo"' >> "$FILE" exec bash To attach: ``/usr/share/python-brickclient/venv/bin/cinder local-attach `` To detach: ``/usr/share/python-brickclient/venv/bin/cinder local-detach `` Alternatively, the same action can be completed manually at the node which does not require setting up of config drive such as: .. code-block:: bash /usr/share/python-brickclient/venv/bin/cinder \ --os-username demo --os-password password \ --os-tenant-name demo --os-project-name demo \ --os-auth-url=http://:5000/v2.0 local-attach diskimage-builder-2.11.0/diskimage_builder/elements/python-brickclient/element-deps000066400000000000000000000000441324273057700305600ustar00rootroot00000000000000package-installs pip-and-virtualenv diskimage-builder-2.11.0/diskimage_builder/elements/python-brickclient/package-installs.yaml000066400000000000000000000002151324273057700323610ustar00rootroot00000000000000libssl-dev: libffi-dev: python-dev: installtype: source dib_python_version: 2 python3-dev: installtype: source dib_python_version: 3 diskimage-builder-2.11.0/diskimage_builder/elements/python-brickclient/pkg-map000066400000000000000000000005271324273057700275400ustar00rootroot00000000000000{ "family": { "redhat": { "python2-dev": "python2-devel", "python3-dev": "python3-devel", "libssl-dev": "openssl-devel", "libffi-dev": "libffi-devel" } }, "default": { "python2-dev": "python2-dev", "python3-dev": "python3-dev", "libssl-dev": "libssl-dev", "libffi-dev": "libffi-dev" } } diskimage-builder-2.11.0/diskimage_builder/elements/python-brickclient/post-install.d/000077500000000000000000000000001324273057700311305ustar00rootroot0000000000000055-brick-client-install000077500000000000000000000017451324273057700352470ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/python-brickclient/post-install.d#!/bin/bash # # Copyright 2016 Hewlett Packard Enterprise Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail PBCDIR=/usr/share/python-brickclient mkdir $PBCDIR # create the virtual environment virtualenv $PBCDIR/venv # Install the required packages inside virtual env $PBCDIR/venv/bin/pip install python-brick-cinderclient-ext ln -s $PBCDIR/venv/bin/python-brickclient /usr/local/bin/python-brickclient diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/000077500000000000000000000000001324273057700250075ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/README.rst000066400000000000000000000001571324273057700265010ustar00rootroot00000000000000============ ramdisk-base ============ Shared functionality required by all of the different ramdisk elements. diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/cleanup.d/000077500000000000000000000000001324273057700266605ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/cleanup.d/99-extract-ramdisk-files000077500000000000000000000004541324273057700332520ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail IMAGE_NAME=${IMAGE_NAME:-'image'} sudo chmod a+r $TMP_MOUNT_PATH/tmp/ramdisk $TMP_MOUNT_PATH/tmp/kernel cp $TMP_MOUNT_PATH/tmp/ramdisk $IMAGE_NAME.initramfs cp $TMP_MOUNT_PATH/tmp/kernel $IMAGE_NAME.kernel diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/extra-data.d/000077500000000000000000000000001324273057700272635ustar00rootroot0000000000000001-inject-ramdisk-build-files000077500000000000000000000006071324273057700344540ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/extra-data.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail export RAMDISK_BUILD_PATH="$TMP_MOUNT_PATH/tmp/ramdisk-build" mkdir -p $RAMDISK_BUILD_PATH for file in common-defaults common-functions ramdisk-defaults ramdisk-functions img-defaults img-functions ; do cp $_LIB/$file $RAMDISK_BUILD_PATH done cp -r $(dirname $0)/scripts $RAMDISK_BUILD_PATH diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/extra-data.d/scripts/000077500000000000000000000000001324273057700307525ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/extra-data.d/scripts/d/000077500000000000000000000000001324273057700311755ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/extra-data.d/scripts/d/init-func000066400000000000000000000167141324273057700330250ustar00rootroot00000000000000# Copyright (c) 2012 NTT DOCOMO, INC. # Copyright 2012 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. source /iscsi-func function configure_vmedia_dir() { VMEDIA_MOUNT_POINT="/vfloppy_mnt" VMEDIA_DIR="/vfloppy" VMEDIA_CONFIG_FILE="parameters.txt" VMEDIA_DEVICE_BY_LABEL="/dev/disk/by-label/ir-vfd-dev" # Wait for some seconds for kernel to finish initializing the # virtual media devices. wait_for 5 1 "ls $VMEDIA_DEVICE_BY_LABEL" if [[ $? != 0 ]]; then # TODO(rameshg87) - 04/29/15 : This block of code is there only # for backward compatibility to older versions of Ironic. Remove this # after 2 releases. Ironic will now assign labels to Virtual Media # devices. SYSFS_DEVICE_MODELS="/sys/class/block/*/device/model" SYSFS_VMEDIA_DEVICE_MODEL="virtual media" grep -q -i "$SYSFS_VMEDIA_DEVICE_MODEL" $SYSFS_DEVICE_MODELS if [[ $? != 0 ]]; then echo "Error finding the virtual floppy device on the node." troubleshoot fi # Find the virtual media device from the kernel sysfs file system. VMEDIA_DEVICE=$(grep -l -i "$SYSFS_VMEDIA_DEVICE_MODEL" $SYSFS_DEVICE_MODELS | awk -F'/' '{print $5}') if [[ -z $VMEDIA_DEVICE ]]; then echo "Error finding the virtual floppy device on the node." troubleshoot fi VMEDIA_DEVICE_FILE="/dev/$VMEDIA_DEVICE" else VMEDIA_DEVICE_FILE="$VMEDIA_DEVICE_BY_LABEL" fi # Mount the virtual media device mkdir -p $VMEDIA_MOUNT_POINT mount $VMEDIA_DEVICE_FILE $VMEDIA_MOUNT_POINT if [[ $? != 0 ]]; then echo "Error mounting virtual media device $VMEDIA_DEVICE_FILE" troubleshoot fi # Copy the contents of the virtual media to a directory in root # filesystem so that we can unmount virtual media soon. cp -rf $VMEDIA_MOUNT_POINT $VMEDIA_DIR umount $VMEDIA_MOUNT_POINT } function get_kernel_parameter() { if [ "$BOOT_METHOD" = "$VMEDIA_BOOT_TAG" ]; then echo $(find_parameter_from_file $1 "$VMEDIA_DIR/$VMEDIA_CONFIG_FILE") else echo $(find_parameter_from_file $1 "/proc/cmdline") fi } function find_parameter_from_file() { local paramname=$1 local filename=$2 for i in `cat $filename`; do case "$i" in ${paramname}=*) echo "${i#${paramname}=}" return 0 ;; ${paramname}) echo "" return 0 ;; *) ;; esac done echo "" return 1 } function string_contains() { local string=$1 local word=$2 if [ "$string" != "${string/$word/}" ]; then return 0 else return 1 fi } function load_modules_by_udev() { udevadm trigger --action=add udevadm settle } function strip_dev() { echo "$1" | sed -e 's:^/dev/::' } function prepend_dev() { case "$1" in /dev/*) echo "$1" ;; *) echo "/dev/$1" ;; esac } function whole_disk_name() { local dev=`prepend_dev "$1"` case "$dev" in /dev/sd*|/dev/hd*|/dev/vd*|/dev/xvd*) echo "$dev" | sed -e 's/[0-9]*$//' ;; /dev/cciss/*) echo "$dev" | sed -e 's/p[0-9]*$//' ;; *) echo "" return 1 ;; esac } function partition_name() { local dev=`prepend_dev "$1"` local part=$2 case "$dev" in /dev/sd*|/dev/hd*|/dev/vd*|/dev/xvd*) echo "${dev}${part}" ;; /dev/cciss/*) echo "${dev}p${part}" ;; *) echo "" return 1 ;; esac } function find_interface() { local mac=$1 local interface interface=$(ip link show | grep -B1 -i $mac | awk '/mtu/ { print $2 }') if [ -n "$interface" ]; then echo "${interface%:}" return 0 else # Note(moshele): try to find the InfinBand interface by # matching the InfinBand MAC upper and lower # 3 octets to the GUID lower and upper 3 octets. local upper_octs=${mac:0:8} local lower_octs=${mac:9:16} ib_re="${upper_octs}:[A-F0-9][A-F0-9]:[A-F0-9][A-F0-9]:${lower_octs}" interface=$(ip link show | grep -B1 -i ${ib_re} | grep mtu | \ awk {'print $2'}) if [ -n "$interface" ]; then echo "${interface%:}" return 0 fi return 1 fi } function set_mac() { local dev=$1 local mac=$2 ip link set "$dev" address "$mac" } function swap_ifname() { local dev=$1 local dev2=$2 if [ "$dev" = "$dev2" ]; then return fi if ip link show "$dev2" >/dev/null; then # dev2 exists # swap device name ip link set "$dev" name "_$dev" ip link set "$dev2" name "$dev" ip link set "_$dev" name "$dev2" else ip link set "$dev" name "$dev2" fi } function partition_exists() { local dev=$1 dev=`strip_dev "$dev"` if tail -n +3 /proc/partitions | grep "$dev" >/dev/null; then return 0 else return 1 fi } function find_disk() { local disks=$1 local dev # find device local OLD_IFS=$IFS IFS=, for i in $disks; do dev=`whole_disk_name "$i"` if partition_exists "$dev"; then dev=`prepend_dev "$dev"` break fi dev="" done IFS=$OLD_IFS if [ -z "$dev" ]; then return 1 fi echo "$dev" return 0 } function check_tgtd_socket() { echo -n "waiting for tgtd socket..." if [ -e /var/run/tgtd.ipc_abstract_namespace.0 ]; then echo "found" return 0 else echo "not found" return 1 fi } wait_for(){ LOOPS=$1 SLEEPTIME=$2 shift ; shift i=0 while [ $i -lt $LOOPS ] ; do i=$((i + 1)) eval "$@" && return 0 || true sleep $SLEEPTIME done return 1 } function troubleshoot() { if [ "$TROUBLESHOOT" != 1 ]; then _DO_TROUBLESHOOT="" _t=0 echo -n "Troubleshooting required, press t to launch shell." while [ $_t -lt 10 ]; do read -n 1 -t 1 _DO_TROUBLESHOOT _t=$(($_t + 1)) if [ "$_DO_TROUBLESHOOT" == "t" ]; then export TROUBLESHOOT=1 break fi echo -n "." done echo "" fi if [ "$TROUBLESHOOT" == 1 ]; then echo "Starting troubleshooting shell." bash fi } function safe_url_encode() { local str=$1 local out="" for (( i=0; i<${#str}; i++ )); do c=${str:$i:1} case $c in [a-zA-Z0-9.-_] ) out+="$c" ;; ' ' ) out+="+" ;; *) #skip it ;; esac done echo "$out" } function err_msg() { message=$1 if [ -z "$FIRST_ERR_MSG" ]; then FIRST_ERR_MSG=$(safe_url_encode "$message") fi echo "$message" } diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/extra-data.d/scripts/init000077500000000000000000000016701324273057700316470ustar00rootroot00000000000000#!/bin/bash # Copyright (c) 2012 NTT DOCOMO, INC. # Copyright 2012 Hewlett-Packard Development Company, L.P. # # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # NOTE(bnemec): We don't want this script to exit on failures because if init # dies then we get a kernel panic on ramdisk boot. # dib-lint: disable=sete setu setpipefail dibdebugtrace echo "init" source /init-func PATH=/sbin:/bin:/usr/bin:/usr/sbin export PATH diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/extra-data.d/scripts/init-end000066400000000000000000000001661324273057700324070ustar00rootroot00000000000000 # Final init steps echo "rebooting" reboot -f echo "failed to reboot" echo "Starting troubleshooting shell." bash diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/init.d/000077500000000000000000000000001324273057700261745ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/init.d/20-init-variables000066400000000000000000000042021324273057700312450ustar00rootroot00000000000000FIRST_ERR_MSG= # To identify that the node has booted from virtual media, a kernel # command-line argument boot_method=vmedia is added in the iso bootloader # configuration file (like isolinux.cfg for isolinux). VMEDIA_BOOT_TAG="vmedia" BOOT_METHOD=$(get_kernel_parameter boot_method) if [ "$BOOT_METHOD" = "$VMEDIA_BOOT_TAG" ]; then # If the node booted from virtual media cdrom, the arguments for the # installation are provided in virtual media floppy. Find out # the virtual media device, mount it and get the information. configure_vmedia_dir fi readonly _BOOTIF_=$(get_kernel_parameter BOOTIF) readonly _IP_=$(get_kernel_parameter ip) # _BOOTIF_ is the hardware type with the MAC address # see http://www.syslinux.org/wiki/index.php/PXELINUX if [[ $_BOOTIF_ =~ ^20.* && ${#_BOOTIF_} == 20 ]]; then # InfiniBand hardware type is 20 with InfiniBand MAC # For example InfiniBand GID: # 80:00:02:08:fe:80:00:00:00:00:00:00:f4:52:14:03:00:3a:16:b1 # is converted to BOOTIF 20:f4:52:14:3a:16:b1 MAC_ADDRESS=$(echo "$_BOOTIF_" | sed -e "s/-/:/g" | \ sed -e "s/^20://g" | tr 'a-f' 'A-F') else # Ethernet BOOTIF hardware type is 01 with MAC MAC_ADDRESS=$(echo "$_BOOTIF_" | sed -e "s/-/:/g" | \ sed -e "s/^01://g" | tr 'a-f' 'A-F') fi readonly BOOT_MAC_ADDRESS=$MAC_ADDRESS # If bootloader did not pass on network info, we fallback to a kernel # parameter to locate the address of the boot server. if [ -n "$_IP_" ]; then readonly BOOT_SERVER=$(echo "$_IP_" | cut -d':' -f2) else readonly BOOT_SERVER=$(get_kernel_parameter boot_server) fi # IP address, netmask, gateway can be set a later point of time if # IP address is assigned by dhcp (for non-pxe boots). BOOT_IP_ADDRESS=$(echo "$_IP_" | cut -d':' -f1) BOOT_NETMASK=$(echo "$_IP_" | cut -d':' -f4) BOOT_GATEWAY=$(echo "$_IP_" | cut -d':' -f3) readonly DISK=$(get_kernel_parameter disk) readonly DEPLOYMENT_ID=$(get_kernel_parameter deployment_id) readonly DEPLOYMENT_KEY=$(get_kernel_parameter deployment_key) readonly ISCSI_TARGET_IQN=$(get_kernel_parameter iscsi_target_iqn) TROUBLESHOOT=$(get_kernel_parameter troubleshoot) diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/init.d/40-check-network-ready000066400000000000000000000002371324273057700322100ustar00rootroot00000000000000if [ -n "$BOOT_SERVER" ]; then echo "pinging to boot server $BOOT_SERVER" wait_for 6 5 ping -c 5 -q "$BOOT_SERVER" > /dev/null fi echo "network ready"diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/post-install.d/000077500000000000000000000000001324273057700276625ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/post-install.d/01-ensure-binaries000077500000000000000000000020721324273057700331220ustar00rootroot00000000000000#!/bin/bash # Ensure that all binaries listed in ramdisk elements, exist if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail export TARGET_DIR="/tmp/in_target.d/" if [ -z $TARGET_DIR ] ; then echo "Target dir not specified" exit 1 fi if [ ! -d $TARGET_DIR ] ; then echo "Unable to find specified directory in target: $TARGET_DIR" bash exit 1 fi BINARY_DEPS= for _FILE in $(ls ${TARGET_DIR}/binary-deps.d/) ; do _FILE="${TARGET_DIR}/binary-deps.d/${_FILE}" if [ -a $_FILE ]; then for _LINE in $(cat $_FILE) ; do BINARY_DEPS="${BINARY_DEPS} $_LINE" done fi done for _BIN in $BINARY_DEPS ; do _LOCATION=$(type -p "$_BIN" || echo "") if [ -z "$_LOCATION" ]; then echo "$_BIN is not found in PATH. Please ensure your elements install it" exit 1 fi done if [ "$BINARY_DEPS" == "" ]; then echo "No binary-deps found" else DEPS_OUTPUT="/etc/dib_binary_deps" echo "Creating binary_deps record at: ${DEPS_OUTPUT}" echo "$BINARY_DEPS" >${DEPS_OUTPUT} fi diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/udev.d/000077500000000000000000000000001324273057700261745ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk-base/udev.d/60-persistent-storage.rules000066400000000000000000000130261324273057700333370ustar00rootroot00000000000000# do not edit this file, it will be overwritten on update # persistent storage links: /dev/disk/{by-id,by-uuid,by-label,by-path} # scheme based on "Linux persistent device names", 2004, Hannes Reinecke # forward scsi device event to corresponding block device ACTION=="change", SUBSYSTEM=="scsi", ENV{DEVTYPE}=="scsi_device", TEST=="block", ATTR{block/*/uevent}="change" ACTION=="remove", GOTO="persistent_storage_end" # enable in-kernel media-presence polling ACTION=="add", SUBSYSTEM=="module", KERNEL=="block", ATTR{parameters/events_dfl_poll_msecs}=="0", ATTR{parameters/events_dfl_poll_msecs}="2000" ACTION=="add", ATTR{removable}=="1", ATTR{events_poll_msecs}=="-1", ATTR{events_poll_msecs}="2000" SUBSYSTEM!="block", GOTO="persistent_storage_end" # skip rules for inappropriate block devices KERNEL=="fd*|mtd*|nbd*|gnbd*|btibm*|dm-*|md*", GOTO="persistent_storage_end" # ignore partitions that span the entire disk TEST=="whole_disk", GOTO="persistent_storage_end" # for partitions import parent information ENV{DEVTYPE}=="partition", IMPORT{parent}="ID_*" # virtio-blk KERNEL=="vd*[!0-9]", ATTRS{serial}=="?*", ENV{ID_SERIAL}="$attr{serial}", SYMLINK+="disk/by-id/virtio-$env{ID_SERIAL}" KERNEL=="vd*[0-9]", ATTRS{serial}=="?*", ENV{ID_SERIAL}="$attr{serial}", SYMLINK+="disk/by-id/virtio-$env{ID_SERIAL}-part%n" # ATA devices using the "scsi" subsystem KERNEL=="sd*[!0-9]|sr*", ENV{ID_SERIAL}!="?*", SUBSYSTEMS=="scsi", ATTRS{vendor}=="ATA", IMPORT{program}="ata_id --export $devnode" # ATA/ATAPI devices (SPC-3 or later) using the "scsi" subsystem KERNEL=="sd*[!0-9]|sr*", ENV{ID_SERIAL}!="?*", SUBSYSTEMS=="scsi", ATTRS{type}=="5", ATTRS{scsi_level}=="[6-9]*", IMPORT{program}="ata_id --export $devnode" # Run ata_id on non-removable USB Mass Storage (SATA/PATA disks in enclosures) KERNEL=="sd*[!0-9]|sr*", ENV{ID_SERIAL}!="?*", ATTR{removable}=="0", SUBSYSTEMS=="usb", IMPORT{program}="ata_id --export $devnode" # Otherwise fall back to using usb_id for USB devices KERNEL=="sd*[!0-9]|sr*", ENV{ID_SERIAL}!="?*", SUBSYSTEMS=="usb", IMPORT{builtin}="usb_id" # scsi devices KERNEL=="sd*[!0-9]|sr*", ENV{ID_SERIAL}!="?*", IMPORT{program}="scsi_id --export --whitelisted -d $devnode", ENV{ID_BUS}="scsi" KERNEL=="cciss*", ENV{DEVTYPE}=="disk", ENV{ID_SERIAL}!="?*", IMPORT{program}="scsi_id --export --whitelisted -d $devnode", ENV{ID_BUS}="cciss" KERNEL=="sd*|sr*|cciss*", ENV{DEVTYPE}=="disk", ENV{ID_SERIAL}=="?*", SYMLINK+="disk/by-id/$env{ID_BUS}-$env{ID_SERIAL}" KERNEL=="sd*|cciss*", ENV{DEVTYPE}=="partition", ENV{ID_SERIAL}=="?*", SYMLINK+="disk/by-id/$env{ID_BUS}-$env{ID_SERIAL}-part%n" # firewire KERNEL=="sd*[!0-9]|sr*", ATTRS{ieee1394_id}=="?*", SYMLINK+="disk/by-id/ieee1394-$attr{ieee1394_id}" KERNEL=="sd*[0-9]", ATTRS{ieee1394_id}=="?*", SYMLINK+="disk/by-id/ieee1394-$attr{ieee1394_id}-part%n" KERNEL=="mmcblk[0-9]", SUBSYSTEMS=="mmc", ATTRS{name}=="?*", ATTRS{serial}=="?*", ENV{ID_NAME}="$attr{name}", ENV{ID_SERIAL}="$attr{serial}", SYMLINK+="disk/by-id/mmc-$env{ID_NAME}_$env{ID_SERIAL}" KERNEL=="mmcblk[0-9]p[0-9]", ENV{ID_NAME}=="?*", ENV{ID_SERIAL}=="?*", SYMLINK+="disk/by-id/mmc-$env{ID_NAME}_$env{ID_SERIAL}-part%n" KERNEL=="mmcblk[0-9]p[0-9]", ENV{PARTNAME}=="?*", SYMLINK+="disk/by-partlabel/$env{PARTNAME}" KERNEL=="mspblk[0-9]", SUBSYSTEMS=="memstick", ATTRS{name}=="?*", ATTRS{serial}=="?*", ENV{ID_NAME}="$attr{name}", ENV{ID_SERIAL}="$attr{serial}", SYMLINK+="disk/by-id/memstick-$env{ID_NAME}_$env{ID_SERIAL}" KERNEL=="mspblk[0-9]p[0-9]", ENV{ID_NAME}=="?*", ENV{ID_SERIAL}=="?*", SYMLINK+="disk/by-id/memstick-$env{ID_NAME}_$env{ID_SERIAL}-part%n" # by-path (parent device path) ENV{DEVTYPE}=="disk", DEVPATH!="*/virtual/*", IMPORT{builtin}="path_id" ENV{DEVTYPE}=="disk", ENV{ID_PATH}=="?*", SYMLINK+="disk/by-path/$env{ID_PATH}" ENV{DEVTYPE}=="partition", ENV{ID_PATH}=="?*", SYMLINK+="disk/by-path/$env{ID_PATH}-part%n" # skip unpartitioned removable media devices from drivers which do not send "change" events ENV{DEVTYPE}=="disk", KERNEL!="sd*|sr*", ATTR{removable}=="1", GOTO="persistent_storage_end" # probe filesystem metadata of optical drives which have a media inserted KERNEL=="sr*", ENV{DISK_EJECT_REQUEST}!="?*", ENV{ID_CDROM_MEDIA_TRACK_COUNT_DATA}=="?*", ENV{ID_CDROM_MEDIA_SESSION_LAST_OFFSET}=="?*", \ IMPORT{builtin}="blkid --offset=$env{ID_CDROM_MEDIA_SESSION_LAST_OFFSET}" # single-session CDs do not have ID_CDROM_MEDIA_SESSION_LAST_OFFSET KERNEL=="sr*", ENV{DISK_EJECT_REQUEST}!="?*", ENV{ID_CDROM_MEDIA_TRACK_COUNT_DATA}=="?*", ENV{ID_CDROM_MEDIA_SESSION_LAST_OFFSET}=="", \ IMPORT{builtin}="blkid --noraid" # probe filesystem metadata of disks KERNEL!="sr*", IMPORT{builtin}="blkid" # watch metadata changes by tools closing the device after writing KERNEL!="sr*", OPTIONS+="watch" # by-label/by-uuid links (filesystem metadata) ENV{ID_FS_USAGE}=="filesystem|other|crypto", ENV{ID_FS_UUID_ENC}=="?*", SYMLINK+="disk/by-uuid/$env{ID_FS_UUID_ENC}" ENV{ID_FS_USAGE}=="filesystem|other", ENV{ID_FS_LABEL_ENC}=="?*", SYMLINK+="disk/by-label/$env{ID_FS_LABEL_ENC}" # by-id (World Wide Name) ENV{DEVTYPE}=="disk", ENV{ID_WWN_WITH_EXTENSION}=="?*", SYMLINK+="disk/by-id/wwn-$env{ID_WWN_WITH_EXTENSION}" ENV{DEVTYPE}=="partition", ENV{ID_WWN_WITH_EXTENSION}=="?*", SYMLINK+="disk/by-id/wwn-$env{ID_WWN_WITH_EXTENSION}-part%n" # by-partlabel/by-partuuid links (partition metadata) ENV{ID_PART_ENTRY_SCHEME}=="gpt", ENV{ID_PART_ENTRY_UUID}=="?*", SYMLINK+="disk/by-partuuid/$env{ID_PART_ENTRY_UUID}" ENV{ID_PART_ENTRY_SCHEME}=="gpt", ENV{ID_PART_ENTRY_NAME}=="?*", SYMLINK+="disk/by-partlabel/$env{ID_PART_ENTRY_NAME}" LABEL="persistent_storage_end" diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk/000077500000000000000000000000001324273057700240775ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk/README.rst000066400000000000000000000015331324273057700255700ustar00rootroot00000000000000======= ramdisk ======= This is the ramdisk element. Almost any user building a ramdisk will want to include this in their build, as it triggers many of the vital functionality from the basic diskimage-builder libraries (such as init script aggregation, busybox population, etc). An example of when one might want to use this toolchain to build a ramdisk would be the initial deployment of baremetal nodes in a TripleO setup. Various tools and scripts need to be injected into a ramdisk that will fetch and apply a machine image to local disks. That tooling/scripting customisation can be easily applied in a repeatable and automatable way, using this element. NOTE: ramdisks require 1GB minimum memory on the machines they are booting. See the top-level README.md of the project, for more information about the mechanisms available to a ramdisk element. diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk/binary-deps.d/000077500000000000000000000000001324273057700265365ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk/binary-deps.d/ramdisk000066400000000000000000000000241324273057700301070ustar00rootroot00000000000000dhclient ip busybox diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk/element-deps000066400000000000000000000000431324273057700264010ustar00rootroot00000000000000deploy-tgtadm pkg-map ramdisk-base diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk/init.d/000077500000000000000000000000001324273057700252645ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk/init.d/10-start-base-system000066400000000000000000000024631324273057700310210ustar00rootroot00000000000000mkdir -p /proc /sys /dev /boot /etc /mnt /lib/modules mount -t proc proc /proc mount -t sysfs none /sys UDEVD= if [ -x "/bin/systemd-udevd" ]; then UDEVD="systemd-udevd" else UDEVD="udevd" fi # udev versions 176 and newer require a different on-disk setup UDEVD_VERSION=$(udevadm --version) if [ "$UDEVD_VERSION" != "" -a $UDEVD_VERSION -gt 175 ]; then echo "Using new-style udevd setup" mount -t devtmpfs none /dev mkdir -p /run mount -t tmpfs -o "nosuid,size=20%,mode=0755" tmpfs /run mkdir -p /run/{lock,udev} else echo "Using old-style udevd setup" mount -t tmpfs none /dev ln -sf /proc/self/fd /dev/fd mknod /dev/null c 1 3 mknod /dev/zero c 1 5 mknod /dev/random c 1 8 mknod /dev/urandom c 1 9 mknod /dev/tty0 c 4 0 mknod /dev/tty1 c 4 1 mknod /dev/tty2 c 4 2 mknod /dev/tty3 c 4 3 mknod /dev/tty4 c 4 4 mknod /dev/tty5 c 4 5 mknod /dev/tty6 c 4 6 mknod /dev/tty7 c 4 7 mknod /dev/tty8 c 4 8 mknod /dev/tty9 c 4 9 mknod /dev/tty c 5 0 mknod -m 0600 /dev/console c 5 1 mknod -m 0666 /dev/ptmx c 5 2 mkdir -p /dev/.udev/data fi echo "starting syslogd" echo '*.* /initlog' > /etc/syslog.conf syslogd klogd echo "starting udevd" $UDEVD --daemon --resolve-names=never echo "load modules" load_modules_by_udev diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk/init.d/30-start-network000066400000000000000000000030071324273057700302530ustar00rootroot00000000000000echo "starting network $BOOT_MAC_ADDRESS" t=0 while ! BOOT_INTERFACE=$(find_interface "$BOOT_MAC_ADDRESS"); do t=`expr "$t" + 5` if [ "$t" -gt 10 ]; then break fi sleep 5 done if [ -z "$BOOT_INTERFACE" ]; then err_msg "Could not find an interface that owns MAC: $BOOT_MAC_ADDRESS" troubleshoot fi readonly BOOT_INTERFACE ifconfig lo 127.0.0.1 up rv=0 ifconfig "$BOOT_INTERFACE" up || rv=1 if [ $rv -ne 0 ]; then sleep 10 rv=0 ifconfig "$BOOT_INTERFACE" up || rv=1 if [ $? -ne 0 ]; then err_msg "Failed to ifconfig up $BOOT_INTERFACE" troubleshoot fi fi # Check if boot IP address was specific or retrieve from DHCP if [ -n "$BOOT_IP_ADDRESS" ]; then ifconfig "$BOOT_INTERFACE" "$BOOT_IP_ADDRESS" netmask "$BOOT_NETMASK" route del default || true route add default gw $BOOT_GATEWAY else dhclient -1 "$BOOT_INTERFACE" if [[ $? == 2 ]]; then echo "Error getting IP address for $BOOT_INTERFACE with MAC \ $BOOT_MAC_ADDRESS" troubleshoot fi BOOT_IP_ADDRESS=$(ifconfig "$BOOT_INTERFACE" | grep 'inet addr:' | \ cut -d: -f2 | awk '{ print $1}') BOOT_NETMASK=$(ifconfig "$BOOT_INTERFACE" | grep 'Mask:' | cut -d':' -f4) BOOT_GATEWAY=$(route -n | grep "$BOOT_INTERFACE" | grep '^0.0.0.0' | \ awk '{print $2}') echo "obtained the following from dhcp: " echo " ip address = $BOOT_IP_ADDRESS" echo " netmask = $BOOT_NETMASK" echo " gateway = $BOOT_GATEWAY" fi diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk/install.d/000077500000000000000000000000001324273057700257675ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk/install.d/20-install-dhcp-client000077500000000000000000000002601324273057700317700ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail DHCP_CLIENT=$(pkg-map --element ramdisk dhcp_client) install-packages $DHCP_CLIENT diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk/install.d/52-ramdisk-install-busybox000077500000000000000000000001701324273057700327260ustar00rootroot00000000000000#!/bin/bash if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail install-packages busybox diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk/pkg-map000066400000000000000000000002661324273057700253620ustar00rootroot00000000000000{ "family":{ "debian":{ "dhcp_client":"isc-dhcp-client" }, "suse":{ "dhcp_client":"dhcp-client" } }, "default":{ "dhcp_client":"dhclient" } } diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk/post-install.d/000077500000000000000000000000001324273057700267525ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ramdisk/post-install.d/99-build-ramdisk000077500000000000000000000035231324273057700316710ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail _LIB="/tmp/ramdisk-build" source $_LIB/common-defaults source $_LIB/img-defaults source $_LIB/ramdisk-defaults source $_LIB/common-functions source $_LIB/img-functions source $_LIB/ramdisk-functions KERNEL_VERSION=${DIB_KERNEL_VERSION:-$(find_kernel_version)} MODULE_DIR=$MODULE_ROOT/lib/modules/$KERNEL_VERSION if [ -f /dib-signed-kernel-version ] ; then . /dib-signed-kernel-version fi if [ -n "${DIB_SIGNED_KERNEL_VERSION:-}" ]; then # Though kernel name is suffixed with efi.signed, modules directory is # without that suffix MOD_KERNEL_NAME=`echo "$DIB_SIGNED_KERNEL_VERSION" |sed "s/\.efi\.signed//g"` MODULE_DIR=$MODULE_ROOT/lib/modules/$MOD_KERNEL_NAME fi FIRMWARE_DIR=$MODULE_ROOT/lib/firmware LIB_UDEV=$LIB_UDEV_ROOT/lib/udev INIT="$_LIB/scripts/init" FUNCTIONS_D="$_LIB/scripts/d" BUSYBOX=${BUSYBOX:-$(type -p busybox)} # NOTE(bnemec): IMAGE_ELEMENT is normally set in disk-image-create, but we're # not using that to build the image here. IMAGE_ELEMENT= mk_build_dir mkdir -p $TMP_BUILD_DIR/mnt export TMP_HOOKS_PATH=/tmp export TMP_MOUNT_PATH=$TMP_BUILD_DIR/mnt echo "building ramdisk in $TMP_MOUNT_PATH" create_ramdisk_base populate_lib populate_busybox populate_init populate_udev SCRIPT_HOME=/tmp/in_target.d/bin TMP_HOOKS_PATH=/tmp/in_target.d run_d ramdisk-install finalise_image save_image /tmp/ramdisk # In the past save_image did this for us. If EXIT handler is not # reset ramdisk image builds fail. trap EXIT cp /boot/vmlinu[zx]-${KERNEL_VERSION} /tmp/kernel if [ -n "${DIB_SIGNED_KERNEL_VERSION:-}" ]; then cp /boot/vmlinu[zx]-${DIB_SIGNED_KERNEL_VERSION} /tmp/kernel fi chmod o+r /tmp/kernel if [ -f /dib-signed-kernel-version ] ; then echo "Removing /dib-signed-kernel-version" rm -f /dib-signed-kernel-version fi diskimage-builder-2.11.0/diskimage_builder/elements/rax-nova-agent/000077500000000000000000000000001324273057700252745ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/rax-nova-agent/README.rst000066400000000000000000000004201324273057700267570ustar00rootroot00000000000000============== rax-nova-agent ============== Images for Rackspace Cloud currently require nova-agent to get networking information. Many of the things here are adapted from: https://developer.rackspace.com/blog/bootstrap-your-qcow-images-for-the-rackspace-public-cloud/ diskimage-builder-2.11.0/diskimage_builder/elements/rax-nova-agent/element-deps000066400000000000000000000000241324273057700275750ustar00rootroot00000000000000source-repositories diskimage-builder-2.11.0/diskimage_builder/elements/rax-nova-agent/install.d/000077500000000000000000000000001324273057700271645ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/rax-nova-agent/install.d/05-xen-tools000077500000000000000000000021471324273057700312700ustar00rootroot00000000000000#!/bin/bash # Copyright (c) 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # # See the License for the specific language governing permissions and # limitations under the License. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail XS_TOOLS_DIR=$(mktemp -d) mount -o loop /tmp/xs-tools.iso $XS_TOOLS_DIR pushd $XS_TOOLS_DIR/Linux case "$DISTRO_NAME" in 'ubuntu'|'debian') # Seriously source versions.deb dpkg -i $XE_GUEST_UTILITIES_PKG_FILE_amd64 ;; 'fedora'|'centos'|'centos7'|'opensuse') source versions.rpm rpm -Uvh $XE_GUEST_UTILITIES_PKG_FILE_x86_64 ;; esac popd umount $XS_TOOLS_DIR diskimage-builder-2.11.0/diskimage_builder/elements/rax-nova-agent/install.d/10-nova-agent000077500000000000000000000062651324273057700314000ustar00rootroot00000000000000#!/bin/bash # Copyright (c) 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # # See the License for the specific language governing permissions and # limitations under the License. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail cd /tmp/nova-agent ./installer.sh cat > /etc/cloud/cloud.cfg.d/80_restart_network.cfg <<'EOF' # Restart networking if it's got no IP. This is to catch if we're on RAX # and the baked in dhcp config didn't work, but is now probably reconfigured # by nova-agent runcmd: - ip addr show eth0 | grep inet || ( ifdown -a ; ifup -a ) EOF case "$DISTRO_NAME" in 'ubuntu'|'debian') # cloud-init / nova-agent sad panda hacks cat > /etc/init/nova-agent.conf <<'EOF' # nova-agent - start before cloud-init and hup network interfaces start on starting cloud-init-local task console output script /etc/init.d/nova-agent start sleep 13 ifdown -a ifup -a end script EOF cat > /etc/init/xe-linux-distribution.conf <<'EOF' # xe-linux-distribution - start the xentools before nova-agent runs start on starting nova-agent task console output script /etc/init.d/xe-linux-distribution start sleep 2 end script EOF # stop nova-agent from starting twice since we start it from upstart now update-rc.d -f nova-agent remove update-rc.d -f xe-linux-distribution remove # console fix for PV Ubuntus cat > /etc/init/hvc0.conf <<'EOF' # hvc0 - getty # # This service maintains a getty on hvc0 from the point the system is # started until it is shut down again. start on stopped rc RUNLEVEL=[2345] stop on runlevel [!2345] respawn exec /sbin/getty -L 115200 hvc0 vt102 EOF ;; 'fedora'|'centos'|'centos7'|'opensuse') cat > /usr/local/bin/run-nova-agent.sh <<'EOF' #!/bin/bash /etc/init.d/nova-agent start sleep 13 ifdown -a ifup -a EOF chmod 0755 /usr/local/bin/run-nova-agent.sh cat > /usr/local/bin/run-xe-linux-distribution.sh <<'EOF' #!/bin/bash /etc/init.d/xe-linux-distribution start sleep 2 EOF chmod 0755 /usr/local/bin/run-xe-linux-distribution.sh cat > /etc/systemd/system/nova-agent.service < /etc/systemd/system/xe-linux-distribution.service <<'EOF' [Unit] Description=xentools agent Before=nova-agent Wants=nova-agent [Service] ExecStart=/usr/local/bin/run-xe-linux-distribution.sh [Install] WantedBy=multi-user.target EOF # stop nova-agent from starting twice since we start it from systemd now chkconfig nova-agent off chkconfig xe-linux-distribution off systemctl enable nova-agent systemctl enable xe-linux-distribution ;; esac diskimage-builder-2.11.0/diskimage_builder/elements/rax-nova-agent/package-installs.yaml000066400000000000000000000000431324273057700313770ustar00rootroot00000000000000cloud-init: phase: pre-install.d diskimage-builder-2.11.0/diskimage_builder/elements/rax-nova-agent/source-repository-nova-agent000066400000000000000000000002521324273057700327700ustar00rootroot00000000000000nova-agent tar /tmp/nova-agent https://f97f271e9fde8f434e2a-fa73b8821af96cee574a861a54d04230.ssl.cf3.rackcdn.com/files/nova-agent/nova-agent-Linux-x86_64-1.39.0.tar.gz . diskimage-builder-2.11.0/diskimage_builder/elements/rax-nova-agent/source-repository-xen-tools000066400000000000000000000002251324273057700326610ustar00rootroot00000000000000xen-tools file /tmp/xs-tools.iso https://f97f271e9fde8f434e2a-fa73b8821af96cee574a861a54d04230.ssl.cf3.rackcdn.com/files/xentools/xs-tools-6.2.0.iso diskimage-builder-2.11.0/diskimage_builder/elements/redhat-common/000077500000000000000000000000001324273057700252025ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/redhat-common/README.rst000066400000000000000000000020551324273057700266730ustar00rootroot00000000000000============= redhat-common ============= Image installation steps common to RHEL, CentOS, and Fedora. Requirements: If used to build an image form a cloud image compress with xz (the default in centos), this element uses "unxz" to decompress the image. Depending on your distro you may need to install either the xz or xz-utils package. Environment Variables --------------------- DIB_LOCAL_IMAGE :Required: No :Default: None :Description: Use the local path of a qcow2 cloud image. This is useful in that you can use a customized or previously built cloud image from diskimage-builder as input. The cloud image does not have to have been built by diskimage-builder. It should be a full disk image, not just a filesystem image. :Example: ``DIB_LOCAL_IMAGE=rhel-guest-image-7.1-20150224.0.x86_64.qcow2`` DIB_DISABLE_KERNEL_CLEANUP :Required: No :Default: 0 :Description: Specify if kernel needs to be cleaned up or not. When set to true, the bits that cleanup old kernels will not be executed. :Example: DIB_DISABLE_KERNEL_CLEANUP=1 diskimage-builder-2.11.0/diskimage_builder/elements/redhat-common/bin/000077500000000000000000000000001324273057700257525ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/redhat-common/bin/extract-image000077500000000000000000000113211324273057700304300ustar00rootroot00000000000000#!/bin/bash # Intended to be called from the root.d cloud-image script as follows: # $TMP_HOOKS_PATH/bin/extract-image $BASE_IMAGE_FILE $BASE_IMAGE_TAR $IMAGE_LOCATION $CACHED_IMAGE if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail BASE_IMAGE_FILE=$1 BASE_IMAGE_TAR=$2 IMAGE_LOCATION=$3 CACHED_IMAGE=$4 CACHED_TAR=$DIB_IMAGE_CACHE/$BASE_IMAGE_TAR DIB_LOCAL_IMAGE=${DIB_LOCAL_IMAGE:-""} TAR_LOCK=$CACHED_TAR.lock function extract_image() { if [ -n "$DIB_OFFLINE" -a -f "$CACHED_TAR" ] ; then echo "Not checking freshness of cached $CACHED_TAR." else if [ -z "$DIB_LOCAL_IMAGE" ]; then echo "Fetching Base Image" # There seems to be some bad Fedora mirrors returning http 404's for the cloud image. # If the image fails to download due to a 404 we retry once. set +e $TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE RV=$? set -e if [ "$RV" == "44" ] ; then $TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE elif [ "$RV" != "0" ] ; then exit 1 fi fi if [ ! -f $CACHED_TAR -o \ $CACHED_IMAGE -nt $CACHED_TAR ] ; then echo "Repacking base image as tarball." WORKING=$(mktemp --tmpdir=${TMP_DIR:-/tmp} -d) EACTION="rm -r $WORKING" trap "$EACTION" EXIT echo "Working in $WORKING" RAW_FILE=$(mktemp --tmpdir=$WORKING XXXXXX.raw) if [ "${CACHED_IMAGE: -3}" == ".xz" ] ; then QCOW2_FILE=$(mktemp --tmpdir=$WORKING XXXXXX.qcow2) # This leaves the old image in place so cache-url wont get it again unxz --stdout $CACHED_IMAGE > $QCOW2_FILE CACHED_IMAGE=$QCOW2_FILE fi qemu-img convert -f qcow2 -O raw $CACHED_IMAGE $RAW_FILE ROOT_PARTITION=p$(sudo kpartx -l $RAW_FILE | awk "/loop[0-9]+p/"|wc -l) sudo udevadm settle # kpartx fails if no /dev/loop* exists, "losetup -f" prints first unused # loop device and creates it if it doesn't exist sudo losetup -f # XXX: Parsing stdout is dangerous, would like a better way to discover # the device used for the image. ROOT_LOOPDEV=$(sudo kpartx -av $RAW_FILE | \ awk "/loop[0-9]+$ROOT_PARTITION/ {print \$3}") # If running inside Docker, make our nodes manually, because udev will not be working. if [ -f /.dockerenv ]; then sudo dmsetup --noudevsync mknodes fi if ! timeout 5 sh -c "while ! [ -e /dev/mapper/$ROOT_LOOPDEV ]; do sleep 1; done"; then echo "Error: Could not find /dev/mapper/$ROOT_LOOPDEV" exit 1 fi EACTION="sudo kpartx -d $RAW_FILE ; $EACTION" trap "$EACTION" EXIT mkdir $WORKING/mnt if [ "xfs" = "$(sudo blkid -o value -s TYPE /dev/mapper/$ROOT_LOOPDEV)" ]; then # mount xfs with nouuid, just in case that uuid is already mounted MOUNTOPTS="-o nouuid" else MOUNTOPTS="" fi sudo mount $MOUNTOPTS /dev/mapper/$ROOT_LOOPDEV $WORKING/mnt EACTION="sudo umount -f $WORKING/mnt ; $EACTION" trap "$EACTION" EXIT # find out if chroot tar has full xattr support if [ 0 == `sudo chroot $WORKING/mnt bin/tar --help | grep -c xattrs-exclude` ]; then TAROPTS="--no-xattrs" else TAROPTS="--xattrs --xattrs-include=* --xattrs-exclude=security.selinux" fi # Chroot in so that we get the correct uid/gid sudo chroot $WORKING/mnt bin/tar $TAROPTS -cz . > $WORKING/tmp.tar mv $WORKING/tmp.tar $CACHED_TAR else echo "Using cached tar from $CACHED_TAR" fi fi # Extract the base image (use --numeric-owner to avoid UID/GID mismatch between # image tarball and host OS e.g. when building Fedora image on an openSUSE host) # Include all xattrs except selinux because the selinux ones cause issues in our # chroot environment, and we restore all of those at the end of the build anyway. echo "Extracting base root image from $CACHED_TAR" sudo tar -C $TARGET_ROOT --numeric-owner --xattrs --xattrs-include='*' --xattrs-exclude='security.selinux' -xzf $CACHED_TAR } ( echo "Getting $TAR_LOCK: $(date)" # Wait up to 20 minutes for another process to download if ! flock -w 1200 9 ; then echo "Did not get $TAR_LOCK: $(date)" exit 1 fi extract_image ) 9> $TAR_LOCK diskimage-builder-2.11.0/diskimage_builder/elements/redhat-common/bin/map-packages000077500000000000000000000075701324273057700302420ustar00rootroot00000000000000#!/usr/local/bin/dib-python # dib-lint: disable=indent # dib-lint indent requirements causes issue with pep8 # Copyright 2012 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from __future__ import print_function import sys # Manually maintained for brevity; consider making this compiled from # distromatch or other rich data sources. # Debian name on the left, Fedora/RHEL on the right. # # !!! DO NOT ADD ANY ENTRIES TO THIS FILE !!! # # This global list has been deprecated by the pkg-map element. New # package mappings should go in pkg-map files inside each element. # package_map = { 'apache2': 'httpd', 'arping': 'iputils', 'augeas-tools': 'augeas', 'build-essential': 'make automake gcc gcc-c++ kernel-devel', 'default-jre': 'java', 'extlinux': 'syslinux-extlinux', 'gearman-job-server': 'gearmand', 'grub-pc': 'grub2-tools grub2', 'libaio1': 'libaio', 'libapache2-mod-wsgi': 'mod_wsgi', 'libc6-dev': 'glibc-devel', 'libmariadb-dev': 'mariadb-devel', 'libffi-dev': 'libffi-devel', 'libldap2-dev': 'python-ldap', 'libmysql-dev': 'mysql++-devel', 'libmysql-java': 'mysql-connector-java', 'libmysqlclient-dev': 'mysql-devel', 'libpq-dev': 'libpqxx-devel', 'libxslt-dev': 'libxslt-devel', 'libsasl2-dev': 'cyrus-sasl-devel', 'libsqlite3-dev': 'libsqlite3x-devel', 'libssl-dev': 'openssl-devel', 'libvirt-bin': 'libvirt', 'libxml2-dev': 'libxml2-devel', 'libz-dev': 'zlib-devel', 'linux-headers-generic': 'kernel-headers', 'linux-image-generic': 'kernel', 'lm-sensors': 'lm_sensors', 'mysql-client-5.5': 'mariadb', 'mysql-server-5.5': 'mariadb-server', 'nagios-plugins-basic': 'nagios-plugins-all', 'nfs-common': 'nfs-utils', 'nfs-kernel-server': 'nfs-utils', 'open-iscsi': 'iscsi-initiator-utils', 'openjdk-7-jre-headless': 'java-1.7.0-openjdk-headless', 'openssh-client': 'openssh-clients', 'openvswitch-common': 'openvswitch', 'openvswitch-switch': 'openvswitch', 'python-dev': 'python-devel', 'python-libvirt': 'libvirt-python', 'python-memcache': 'python-memcached', 'python-mysqldb': 'MySQL-python', 'python-numpy': 'numpy', 'python-pyopenssl': 'pyOpenSSL', 'python-xattr': 'pyxattr', 'qemu-utils': 'qemu-img', 'qpid-client': 'qpid-cpp-client', 'qpidd': 'qpid-cpp-server', 'snmp-mibs-downloader': '', 'snmpd': 'net-snmp', 'stunnel4': 'stunnel', 'tftpd-hpa': 'tftp-server', 'tgt': 'scsi-target-utils', 'vlan': 'vconfig', # openstack mappings 'openstack-neutron-dhcp-agent': 'openstack-neutron', } deprecated = [] for arg in sys.argv[1:]: if arg not in package_map and arg.endswith('-dev'): # convert -dev into devel converted = '%s%s' % (arg, 'el') deprecated.append((arg, converted)) print(converted) else: converted = package_map.get(arg, arg) if converted != arg: deprecated.append((arg, converted)) print(converted) if deprecated: print("WARNING: The following packages were re-mapped by " "redhat-common map-packages\n" "They should be converted to pkg-map:", file=sys.stderr) for arg, converted in deprecated: print(" %s -> %s" % (arg, converted), file=sys.stderr) sys.exit(0) # Tell emacs to use python-mode # Local variables: # mode: python # End: diskimage-builder-2.11.0/diskimage_builder/elements/redhat-common/element-deps000066400000000000000000000000311324273057700275010ustar00rootroot00000000000000package-installs sysprep diskimage-builder-2.11.0/diskimage_builder/elements/redhat-common/environment.d/000077500000000000000000000000001324273057700277705ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/redhat-common/environment.d/50-redhat-common000066400000000000000000000001031324273057700326640ustar00rootroot00000000000000export DIB_DISABLE_KERNEL_CLEANUP=${DIB_DISABLE_KERNEL_CLEANUP:-0} diskimage-builder-2.11.0/diskimage_builder/elements/redhat-common/finalise.d/000077500000000000000000000000001324273057700272165ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/redhat-common/finalise.d/01-clean-old-kernels000077500000000000000000000010741324273057700326630ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail if [ $DIB_DISABLE_KERNEL_CLEANUP -ne 0 ]; then echo "Skipping kernel cleanup as configured" exit 0 fi YUM=${YUM:-yum} if [[ ${YUM} == "dnf" ]]; then # cribbed from # http://dnf.readthedocs.org/en/latest/cli_vs_yum.html _old_kernels="$(dnf repoquery --installonly --latest-limit=-1 -q)" if [[ -n "${_old_kernels}" ]]; then dnf remove -y ${_old_kernels} fi else install-packages yum-utils package-cleanup --oldkernels -y --count=1 fi diskimage-builder-2.11.0/diskimage_builder/elements/redhat-common/package-installs.yaml000066400000000000000000000005051324273057700313100ustar00rootroot00000000000000# Install any packages in this file that may not be in the base cloud # image but could reasonably be expected lsof: tcpdump: traceroute: which: gettext: phase: pre-install.d # selinux-targeted policy and related tools selinux-policy: selinux-policy-targeted: libselinux-python: policycoreutils: policycoreutils-python: diskimage-builder-2.11.0/diskimage_builder/elements/rhel-common/000077500000000000000000000000001324273057700246655ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/rhel-common/README.rst000066400000000000000000000237531324273057700263660ustar00rootroot00000000000000=========== rhel-common =========== This element contains the common installation steps between RHEL os releases. RHEL Registration ----------------- This element provides functionality for registering RHEL images during the image build process with the disk-image-create script from diskimage-builder. The RHEL image will register itself with either the hosted Red Hat Customer Portal or Satellite to enable software installation from official repositories. After the end of the image creation process, the image will unregister itself so an entitlement will not be decremented from the account. SECURITY WARNING: ----------------- While the image building workflow will allow you to register with a username and password combination, that feature is deprecated in the boot process via Heat as it will expose your username and password in clear text for anyone that has rights to run heat stack-show. A compromised username and password can be used to login to the Red Hat Customer Portal or an instance of Satellite. An activation key can only be used for registration purposes using the subscription-manager command line tool and is considered a lower security risk. IMPORTANT NOTE: ---------------- The 00-rhsm script is specific to RHEL6. If you use the REG\_ variables to use with RHEL7, you do not need to set any DIB_RHSM variables. The scripts named with "rhel-registration" have not been developed or tested for RHEL6. For information on building RHEL6 images, please see the rhel element README. Environment Variables For Image Creation ---------------------------------------- The following environment variables are used for registering a RHEL instance with either the Red Hat Customer Portal or Satellite 6. #### REG\_ACTIVATION\_KEY Attaches existing subscriptions as part of the registration process. The subscriptions are pre-assigned by a vendor or by a systems administrator using Subscription Asset Manager. #### REG\_AUTO\_ATTACH Automatically attaches the best-matched compatible subscription. This is good for automated setup operations, since the system can be configured in a single step. #### REG\_BASE\_URL Gives the hostname of the content delivery server to use to receive updates. Both Customer Portal Subscription Management and Subscription Asset Manager use Red Hat's hosted content delivery services, with the URL https://cdn.redhat.com. Since Satellite 6 hosts its own content, the URL must be used for systems registered with Satellite 6. #### REG\_ENVIRONMENT Registers the system to an environment within an organization. #### REG\_FORCE Registers the system even if it is already registered. Normally, any register operations will fail if the machine is already registered. #### REG\_HALT\_UNREGISTER At the end of the image build process, the element runs a cleanup script that will unregister it from the system it registered with. There are some cases when building an image where you may want to stop this from happening so you can verify the registration or to build a one off-image where the boot-time registration will not be enabled. Set this value to '1' to stop the unregistration process. #### REG\_MACHINE\_NAME Sets the name of the system to be registered. This defaults to be the same as the hostname. #### REG\_METHOD Sets the method of registration. Use "portal" to register a system with the Red Hat Customer Portal. Use "satellite" to register a system with Red Hat Satellite 6. Use "disable" to skip the registration process. #### REG\_ORG Gives the organization to which to join the system. #### REG\_POOL\_ID The pool ID is listed with the product subscription information, which is available from running the list subcommand of subscription-manager. #### REG\_PASSWORD Gives the password for the user account. #### REG\_RELEASE Sets the operating system minor release to use for subscriptions for the system. Products and updates are limited to that specific minor release version. This is used only used with the REG_AUTO_ATTACH option. Possible values for this include 5Server, 5.7, 5.8, 5.9, 5.10, 6.1,...6.6, 7.0. It will change over time as new releases come out. There are also variants 6Server, 6Client, 6Workstation, 7Server, etc. #### REG\_REPOS A single string representing a list of repository names separated by a comma (No spaces). Each of the repositories in this string are enabled through subscription manager. Once you've attached a subscription, you can find available repositories by running subscription-manager repos --list. #### REG\_SERVER\_URL Gives the hostname of the subscription service to use. The default is for Customer Portal Subscription Management, subscription.rhn.redhat.com. If this option is not used, the system is registered with Customer Portal Subscription Management. #### REG\_SERVICE\_LEVEL Sets the service level to use for subscriptions on that machine. This is only used with the REG_AUTO_ATTACH option. #### REG\_USER Gives the content server user account name. #### REG\_TYPE Sets what type of consumer is being registered. The default is system, which is applicable to both physical systems and virtual guests. Other types include hypervisor for virtual hosts, person, domain, rhui, and candlepin for some subscription management applications. Image Build Registration Examples ------------------------------------ To register with Satellite 6, a common example would be to set the following variables: REG_SAT_URL='http://my-sat06.server.org' REG_ORG='tripleo' REG_ENV='Library' REG_USER='tripleo' REG_PASSWORD='tripleo' REG_METHOD=satellite To register with the Red Hat Customer Portal, a common example would be to set the following variables: REG_REPOS='rhel-7-server-optional-rpms,rhel-7-server-extras-rpms' REG_AUTO_ATTACH=true REG_USER='tripleo' REG_PASSWORD='tripleo' REG_METHOD=portal Configuration ------------- Heat metadata can be used to configure the rhel-common element. rh_registration: activation_key: # Attaches existing subscriptions as part of the registration # process. The subscriptions are pre-assigned by a vendor or by # a systems administrator using Subscription Asset Manager. auto_attach: 'true' # Automatically attaches the best-matched compatible subscription. # This is good for automated setup operations, since the system can # be configured in a single step. base_url: # Gives the hostname of the content delivery server to use to # receive updates. Both Customer Portal Subscription Management # and Subscription Asset Manager use Red Hat's hosted content # delivery services, with the URL https://cdn.redhat.com. Since # Satellite 6 hosts its own content, the URL must be used for # systems registered with Satellite 6. environment: # Registers the system to an environment within an organization. force: # Registers the system even if it is already registered. Normally, # any register operations will fail if the machine is already # registered. machine_name: # Sets the name of the system to be registered. This defaults to be # the same as the hostname. org: # Gives the organization to which to join the system. password: # DEPRECATED # Gives the password for the user account. release: # Sets the operating system minor release to use for subscriptions # for the system. Products and updates are limited to that specific # minor release version. This is only used with the auto_attach # option. repos: # A single string representing a list of repository names separated by a # comma (No spaces). Each of the repositories in this string are enabled # through subscription manager. satellite_url: # The url of the Satellite instance to register with. Required for # Satellite registration. server_url: # Gives the hostname of the subscription service to use. The default # is for Customer Portal Subscription Management, # subscription.rhn.redhat.com. If this option is not used, the system # is registered with Customer Portal Subscription Management. service_level: # Sets the service level to use for subscriptions on that machine. # This is only used with the auto_attach option. user: # DEPRECATED # Gives the content server user account name. type: # Sets what type of consumer is being registered. The default is # "system", which is applicable to both physical systems and virtual # guests. Other types include "hypervisor" for virtual hosts, # "person", "domain", "rhui", and "candlepin" for some subscription # management applications. method: # Sets the method of registration. Use "portal" to register a # system with the Red Hat Customer Portal. Use "satellite" to # register a system with Red Hat Satellite 6. Use "disable" to # skip the registration process. Configuration Registration Examples ------------------------------------ To register with Satellite 6, a common example would be to use the following metadata:: { "rh_registration":{ "satellite_url": "http://my-sat06.server.org", "org": "tripleo", "environment": "Library", "activation_key": "my-key-SQQkh4", "method":"satellite", "repos": "rhel-ha-for-rhel-7-server-rpms" } } To register with the Red Hat Customer Portal, a common example would be to use the following metadata:: { "rh_registration":{ "repos":"rhel-7-server-optional-rpms,rhel-7-server-extras-rpms", "auto_attach":true, "activation_key": "my-key-SQQkh4", "org": "5643002", "method":"portal" } } diskimage-builder-2.11.0/diskimage_builder/elements/rhel-common/finalise.d/000077500000000000000000000000001324273057700267015ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/rhel-common/finalise.d/60-unregister000077500000000000000000000016161324273057700312450ustar00rootroot00000000000000#!/bin/bash if [ "${DIB_DEBUG_TRACE:-1}" -gt 0 ]; then set -x fi set -eu set -o pipefail # When building an image with diskimage-builder, you will need to register # with either the customer portal or a satellite instance in order to enable # repos for software installation. In most cases, it is desirable to # unregister a system when diskimage-builder is completing so the customer # entitlements are not decremented. This option is here for the case when # a single image needs to be built and deployed as a single instance or for # debugging purposes so you can check the portal or satellite to see what was # registered. Set REG_HALT_UNREGISTER to any non-null value to enable this # functionality. case "${REG_METHOD:-}" in portal | satellite) if [ -z "${REG_HALT_UNREGISTER:-}" ]; then subscription-manager unregister fi ;; disable) ;; esac diskimage-builder-2.11.0/diskimage_builder/elements/rhel-common/install.d/000077500000000000000000000000001324273057700265555ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/rhel-common/install.d/10-openstack-selinux-rhel000077500000000000000000000004401324273057700333230ustar00rootroot00000000000000#!/bin/bash # Install openstack-selinux if it is available # dib-lint: disable=sete if [ "${DIB_DEBUG_TRACE:-1}" -gt 0 ]; then set -x fi set -u set -o pipefail HAS_PACKAGE=$(yum list | grep openstack-selinux) if [ -n "$HAS_PACKAGE" ]; then install-packages openstack-selinux fi diskimage-builder-2.11.0/diskimage_builder/elements/rhel-common/os-refresh-config/000077500000000000000000000000001324273057700302055ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/rhel-common/os-refresh-config/pre-configure.d/000077500000000000000000000000001324273057700331745ustar00rootroot0000000000000006-rhel-registration000077500000000000000000000124101324273057700367260ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/rhel-common/os-refresh-config/pre-configure.d#!/bin/bash # dib-lint: disable=dibdebugtrace set -eu set -o pipefail OK=/mnt/state/var/lib/rhsm/rhsm.ok if [ -e $OK ] ; then exit 0 fi REG_ACTIVATION_KEY="$(os-apply-config --key rh_registration.activation_key --type raw --key-default '')" REG_AUTO_ATTACH="$(os-apply-config --key rh_registration.auto_attach --type raw --key-default 'true')" REG_BASE_URL="$(os-apply-config --key rh_registration.base_url --type raw --key-default '')" REG_ENVIRONMENT="$(os-apply-config --key rh_registration.environment --type raw --key-default '')" REG_FORCE="$(os-apply-config --key rh_registration.force --type raw --key-default '')" REG_MACHINE_NAME="$(os-apply-config --key rh_registration.machine_name --type raw --key-default '')" REG_ORG="$(os-apply-config --key rh_registration.org --type raw --key-default '')" REG_PASSWORD="$(os-apply-config --key rh_registration.password --type raw --key-default '')" REG_POOL_ID="$(os-apply-config --key rh_registration.poolid --type raw --key-default '')" REG_RELEASE="$(os-apply-config --key rh_registration.release --type raw --key-default '')" REG_REPOS="$(os-apply-config --key rh_registration.repos --type raw --key-default '')" REG_SAT_URL="$(os-apply-config --key rh_registration.satellite_url --type raw --key-default '')" REG_SERVER_URL="$(os-apply-config --key rh_registration.server_url --type raw --key-default '')" REG_SERVICE_LEVEL="$(os-apply-config --key rh_registration.service_level --type raw --key-default '')" REG_USER="$(os-apply-config --key rh_registration.user --type raw --key-default '')" REG_TYPE="$(os-apply-config --key rh_registration.type --type raw --key-default '')" REG_METHOD="$(os-apply-config --key rh_registration.method --type raw --key-default '')" opts= attach_opts= repos="repos --enable rhel-7-server-rpms" satellite_repo="rhel-7-server-rh-common-rpms" if [ -n "${REG_AUTO_ATTACH:-}" ]; then opts="$opts --auto-attach" if [ -n "${REG_SERVICE_LEVEL:-}" ]; then opts="$opts --servicelevel $REG_SERVICE_LEVEL" fi if [ -n "${REG_RELEASE:-}" ]; then opts="$opts --release=$REG_RELEASE" fi else if [ -n "${REG_SERVICE_LEVEL:-}" ]; then echo "WARNING: REG_SERVICE_LEVEL set without REG_AUTO_ATTACH." fi if [ -n "${REG_RELEASE:-}" ]; then echo "WARNING: REG_RELEASE set without REG_AUTO_ATTACH." fi if [ -n "${REG_POOL_ID:-}" ]; then attach_opts="$attach_opts --pool=$REG_POOL_ID" fi fi if [ -n "${REG_BASE_URL:-}" ]; then opts="$opts --baseurl=$REG_BASE_URL" fi if [ -n "${REG_ENVIRONMENT:-}" ]; then opts="$opts --env=$REG_ENVIRONMENT" fi if [ -n "${REG_FORCE:-}" ]; then opts="$opts --force" fi if [ -n "${REG_SERVER_URL:-}" ]; then opts="$opts --serverurl=$REG_SERVER_URL" fi if [ -n "${REG_ACTIVATION_KEY:-}" ]; then opts="$opts --activationkey=$REG_ACTIVATION_KEY" if [ -z "${REG_ORG:-}" ]; then echo "WARNING: REG_ACTIVATION_KEY set without REG_ORG." fi else echo "WARNING: Support for registering with a username and password is deprecated." echo "Please use activation keys instead. See the README for more information." if [ -n "${REG_PASSWORD:-}" ]; then opts="$opts --password $REG_PASSWORD" fi if [ -n "${REG_USER:-}" ]; then opts="$opts --username $REG_USER" fi fi if [ -n "${REG_MACHINE_NAME:-}" ]; then opts="$opts --name $REG_MACHINE_NAME" fi if [ -n "${REG_ORG:-}" ]; then opts="$opts --org=$REG_ORG" fi if [ -n "${REG_REPOS:-}" ]; then for repo in $(echo $REG_REPOS | tr ',' '\n'); do repos="$repos --enable $repo" done fi if [ -n "${REG_TYPE:-}" ]; then opts="$opts --type=$REG_TYPE" fi sanitized_opts=$(echo "$opts" | sed 's/--password \([^ ]*\)/--password ***/g') sanitized_opts=$(echo "$sanitized_opts" | sed 's/--activationkey=\([^ ]*\)/--activationkey=***/g') case "${REG_METHOD:-}" in portal) echo "Registering with options: $sanitized_opts" subscription-manager register $opts if [ -z "${REG_AUTO_ATTACH:-}" -a -z "${REG_ACTIVATION_KEY:-}" ]; then echo "Attaching with options: $attach_opts" subscription-manager attach $attach_opts fi echo "Enabling repos: $repos" subscription-manager $repos ;; satellite) # Save an unmodified copy of the repo list for logging user_repos=$repos repos="$repos --enable ${satellite_repo}" echo "Installing satellite dependencies" rpm -Uvh "$REG_SAT_URL/pub/katello-ca-consumer-latest.noarch.rpm" || true echo "Registering with options: $sanitized_opts" subscription-manager register $opts echo "Disabling all previous repos" subscription-manager repos --disable=\* echo "Enabling repos: $user_repos" subscription-manager $repos echo "Installing katello-agent" yum install -y katello-agent || true # needed for errata reporting to satellite6 katello-package-upload echo "Disabling satellite repo because it is no longer needed" subscription-manager repos --disable ${satellite_repo} ;; disable) echo "Disabling RHEL registration" ;; *) echo "WARNING: only 'portal', 'satellite', and 'disable' are valid values for REG_METHOD." exit 0 esac mkdir -p $(dirname $OK) touch $OK diskimage-builder-2.11.0/diskimage_builder/elements/rhel-common/pre-install.d/000077500000000000000000000000001324273057700273415ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/rhel-common/pre-install.d/00-rhel-registration000077500000000000000000000065261324273057700331570ustar00rootroot00000000000000#!/bin/bash # This script deals quite a bit with passwords, which we don't ever want # included in trace output # dib-lint: disable=dibdebugtrace set -eu set -o pipefail opts= attach_opts= repos="repos --enable rhel-7-server-rpms" satellite_repo="rhel-7-server-rh-common-rpms" if [ -n "${REG_AUTO_ATTACH:-}" ]; then opts="$opts --auto-attach" if [ -n "${REG_SERVICE_LEVEL:-}" ]; then opts="$opts --servicelevel $REG_SERVICE_LEVEL" fi if [ -n "${REG_RELEASE:-}" ]; then opts="$opts --release=$REG_RELEASE" fi else if [ -n "${REG_SERVICE_LEVEL:-}" ]; then echo "WARNING: REG_SERVICE_LEVEL set without REG_AUTO_ATTACH." fi if [ -n "${REG_RELEASE:-}" ]; then echo "WARNING: REG_RELEASE set without REG_AUTO_ATTACH." fi if [ -n "${REG_POOL_ID:-}" ]; then attach_opts="$attach_opts --pool=$REG_POOL_ID" fi fi if [ -n "${REG_BASE_URL:-}" ]; then opts="$opts --baseurl=$REG_BASE_URL" fi if [ -n "${REG_ENVIRONMENT:-}" ]; then opts="$opts --env=$REG_ENVIRONMENT" fi if [ -n "${REG_FORCE:-}" ]; then opts="$opts --force" fi if [ -n "${REG_SERVER_URL:-}" ]; then opts="$opts --serverurl=$REG_SERVER_URL" fi if [ -n "${REG_ACTIVATION_KEY:-}" ]; then opts="$opts --activationkey=$REG_ACTIVATION_KEY" if [ -z "${REG_ORG:-}" ]; then echo "WARNING: REG_ACTIVATION_KEY set without REG_ORG." fi else if [ -n "${REG_PASSWORD:-}" ]; then opts="$opts --password $REG_PASSWORD" fi if [ -n "${REG_USER:-}" ]; then opts="$opts --username $REG_USER" fi fi if [ -n "${REG_MACHINE_NAME:-}" ]; then opts="$opts --name $REG_MACHINE_NAME" fi if [ -n "${REG_ORG:-}" ]; then opts="$opts --org=$REG_ORG" fi if [ -n "${REG_REPOS:-}" ]; then for repo in $(echo $REG_REPOS | tr ',' '\n'); do repos="$repos --enable $repo" done fi if [ -n "${REG_TYPE:-}" ]; then opts="$opts --type=$REG_TYPE" fi sanitized_opts=$(echo "$opts" | sed 's/--password \([^ ]*\)/--password ***/g') sanitized_opts=$(echo "$sanitized_opts" | sed 's/--activationkey=\([^ ]*\)/--activationkey=***/g') case "${REG_METHOD:-}" in portal) echo "Registering with options: $sanitized_opts" subscription-manager register $opts if [ -z "${REG_AUTO_ATTACH:-}" -a -z "${REG_ACTIVATION_KEY:-}" ]; then echo "Attaching with options: $attach_opts" subscription-manager attach $attach_opts fi echo "Disabling all previous repos" subscription-manager repos --disable=\* echo "Enabling repos: $repos" subscription-manager $repos ;; satellite) # Save an unmodified copy of the repo list for logging user_repos=$repos repos="$repos --enable ${satellite_repo}" echo "Installing satellite dependencies" rpm -Uvh "$REG_SAT_URL/pub/katello-ca-consumer-latest.noarch.rpm" || true echo "Registering with options: $sanitized_opts" subscription-manager register $opts echo "Disabling all previous repos" subscription-manager repos --disable=\* echo "Enabling repos: $user_repos" subscription-manager $repos ;; disable) echo "Disabling RHEL registration" ;; *) echo "WARNING: only 'portal', 'satellite', and 'disable' are valid values for REG_METHOD." exit 0 esac diskimage-builder-2.11.0/diskimage_builder/elements/rhel-common/pre-install.d/00-rhsm000077500000000000000000000044421324273057700304610ustar00rootroot00000000000000#!/bin/bash # This script is for RHEL6 use only. For RHEL7, please see the readme for # documentation on which scripts are used for registration and the variables # needed to enable the registration process. if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail export DIB_RHSM_USER=${DIB_RHSM_USER:-} if [ -n "${DIB_RHSM_USER:-}" ] && [ -n "${DIB_RHSM_PASSWORD:-}" ] ; then opts="--force" if [[ -n "${DIB_SAT_KEY:-}" ]]; then opts="$opts --activationkey ${DIB_SAT_KEY}" else opts="$opts --username ${DIB_RHSM_USER} --password ${DIB_RHSM_PASSWORD}" fi if [[ -n "${DIB_SAT_URL:-}" ]]; then if [[ "${DIB_REG_TYPE:-}" == "rhn" ]]; then opts="$opts --serverUrl ${DIB_SAT_URL}" else opts="$opts --serverurl ${DIB_SAT_URL} --sslCACert /usr/share/rhn/RHN-ORG-TRUSTED-SSL-CERT" fi fi if [[ -n "${DIB_SAT_CERT_RPM_URL:-}" ]]; then yum install -y ${DIB_SAT_CERT_RPM_URL} fi if [[ "${DIB_REG_TYPE:-}" == "rhn" ]]; then rhnreg_ks $opts --norhnsd sleep 1 # optional channel required for diskimage-builder dependency channels="-a -c rhel-x86_64-server-optional-6" if [[ -n "${DIB_RHN_CHANNELS:-}" ]]; then for chan in $DIB_RHN_CHANNELS; do channels="$channels -a -c $chan" done fi rhn-channel --user=$DIB_RHSM_USER --password=$DIB_RHSM_PASSWORD $channels rhn-channel -l else subscription-manager register $opts # wait a second to ensure consumer certificate is finished writing to disk sleep 1 if [ -z ${DIB_RHSM_POOL:-} ]; then subscription-manager attach --auto else subscription-manager attach --pool $DIB_RHSM_POOL fi # optional repo required for diskimage-builder dependency if [ "$DISTRO_NAME" == "rhel7" ]; then repos="--enable rhel-7-server-optional-rpms" else repos="--enable rhel-6-server-optional-rpms" fi if [[ -n "${DIB_RHSM_REPOS:-}" ]]; then for repo in $DIB_RHSM_REPOS; do repos="$repos --enable $repo" done fi subscription-manager repos $repos subscription-manager repos --list fi fi diskimage-builder-2.11.0/diskimage_builder/elements/rhel-common/pre-install.d/10-rhel-blacklist000077500000000000000000000004401324273057700324030ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # This is packages that are required for installation by other # elements (like base) but aren't actually available for this # platform. This is a bit of a hack echo "dkms" > /tmp/yum-blacklist diskimage-builder-2.11.0/diskimage_builder/elements/rhel7/000077500000000000000000000000001324273057700234665ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/rhel7/README.rst000066400000000000000000000022441324273057700251570ustar00rootroot00000000000000===== rhel7 ===== Use RHEL 7 cloud images as the baseline for built disk images. Because RHEL 7 base images are not publicly available, it is necessary to first download the RHEL 7 cloud image from the Red Hat Customer Portal and pass the path to the resulting file to disk-image-create as the ``DIB_LOCAL_IMAGE`` environment variable. The cloud image can be found at (login required): https://access.redhat.com/downloads/content/69/ver=/rhel---7/7.1/x86_64/product-downloads Then before running the image build, define DIB_LOCAL_IMAGE (replace the file name with the one downloaded, if it differs from the example): .. code-block:: bash export DIB_LOCAL_IMAGE=rhel-guest-image-7.1-20150224.0.x86_64.qcow2 The downloaded file will then be used as the basis for any subsequent image builds. For further details about building RHEL 7 images, see the rhel-common and redhat-common element README files. Environment Variables --------------------- DIB_LOCAL_IMAGE :Required: Yes :Default: None :Description: The RHEL 7 base image you have downloaded. See the element description above for more details. :Example: ``DIB_LOCAL_IMAGE=/tmp/rhel7-cloud.qcow2`` diskimage-builder-2.11.0/diskimage_builder/elements/rhel7/element-deps000066400000000000000000000001071324273057700257710ustar00rootroot00000000000000cache-url redhat-common rhel-common rpm-distro source-repositories yum diskimage-builder-2.11.0/diskimage_builder/elements/rhel7/element-provides000066400000000000000000000000211324273057700266640ustar00rootroot00000000000000operating-system diskimage-builder-2.11.0/diskimage_builder/elements/rhel7/environment.d/000077500000000000000000000000001324273057700262545ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/rhel7/environment.d/10-rhel7-distro-name.bash000066400000000000000000000000311324273057700325640ustar00rootroot00000000000000export DISTRO_NAME=rhel7 diskimage-builder-2.11.0/diskimage_builder/elements/rhel7/root.d/000077500000000000000000000000001324273057700246735ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/rhel7/root.d/10-rhel7-cloud-image000077500000000000000000000024461324273057700302520ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail [ -n "$ARCH" ] [ -n "$TARGET_ROOT" ] if [[ "amd64 x86_64" =~ "$ARCH" ]]; then ARCH="x86_64" elif [[ "ppc64le" =~ "$ARCH" ]]; then # We don't need to do anything here other than avoid the else clause : else echo 'rhel7 root element only supports x86_64 and ppc64le values for $ARCH' exit 1 fi DIB_LOCAL_IMAGE=${DIB_LOCAL_IMAGE:-""} if [ -n "$DIB_LOCAL_IMAGE" ]; then IMAGE_LOCATION=$DIB_LOCAL_IMAGE # No need to copy a local image into the cache directory, so just specify # the cached path as the original path. CACHED_IMAGE=$IMAGE_LOCATION BASE_IMAGE_FILE=`basename $DIB_LOCAL_IMAGE` BASE_IMAGE_TAR=$BASE_IMAGE_FILE.tgz else if [ -z "${BASE_IMAGE_FILE:-}" -o -z "${DIB_CLOUD_IMAGES:-}" ]; then echo "No source for a base image file configured." echo "See rhel7 element readme for details on how to obtain and use a base image." exit 1 fi DIB_RELEASE=${DIB_RELEASE:-latest} BASE_IMAGE_TAR=$DIB_RELEASE-rhel-server-$ARCH-latest.tgz IMAGE_LOCATION=$DIB_CLOUD_IMAGES/$BASE_IMAGE_FILE CACHED_IMAGE=$DIB_IMAGE_CACHE/$BASE_IMAGE_FILE fi $TMP_HOOKS_PATH/bin/extract-image $BASE_IMAGE_FILE $BASE_IMAGE_TAR $IMAGE_LOCATION $CACHED_IMAGE diskimage-builder-2.11.0/diskimage_builder/elements/rpm-distro/000077500000000000000000000000001324273057700245455ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/rpm-distro/cleanup.d/000077500000000000000000000000001324273057700264165ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/rpm-distro/cleanup.d/99-selinux-fixfiles-restore000077500000000000000000000054271324273057700335720ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail # parser isn't smart enough to figure out \ # dib-lint: disable=safe_sudo # Here be dragons ... a previous dragon slayer helpfully pointed out in # http://www.spinics.net/lists/selinux/msg17379.html # # Not all of the contexts defined by the offline system's # file_contexts may be valid under the policy of the host on which # you are running (e.g. if they run different distributions or even # different releases of the same distribution), which will normally # prevent setting those contexts (the kernel won't recognize them). # If you have this issue, you'll need to run setfiles as root in a # special domain, setfiles_mac_t, that is allowed to set contexts # unknown to the host policy, and likely chrooted so that it doesn't # ask the kernel whether the contexts are valid via # /sys/fs/selinux/context. That is how livecd-creator supported # creating images for other releases. # One issue you might see without fixing selinux file labels is sshd # will run in the kernel_t domain instead of the sshd_t domain, making # ssh connections fail with "Unable to get valid context for " # error message. Other failures will occur too. # XXX: is it really valid to build rpm-distros without this? if [[ ! -f ${TARGET_ROOT}/etc/selinux/targeted/contexts/files/file_contexts ]]; then echo "No selinux policy found in chroot, skipping..." exit 0 fi if [[ ! -x ${TARGET_ROOT}/usr/sbin/setfiles ]]; then echo "Can not find setfiles in chroot!" exit 1 fi # If we're on a selinux system, enable permissive mode for # setfiles_mac_t so we can relabel within the chroot without concern # for whatever policy is in the host kernel. We will run under # "runcon" to specifically allow this _runcon="" if [[ -d /sys/fs/selinux ]]; then sudo semanage permissive -a setfiles_mac_t _runcon="runcon -t setfiles_mac_t -- " fi # setfiles in > Fedora 26 added this flag: # do not read /proc/mounts to obtain a list of # non-seclabel mounts to be excluded from relabeling # checks. Setting this option is useful where there is # a non-seclabel fs mounted with a seclabel fs # this describes our situation of being on a loopback device on # an ubuntu system, say. See also # https://bugzilla.redhat.com/show_bug.cgi?id=1472709 _dash_m="" if [[ $DISTRO_NAME == "fedora" && $DIB_RELEASE -ge 26 ]]; then _dash_m+="-m" fi IFS='|' read -ra SPLIT_MOUNTS <<< "$DIB_MOUNTPOINTS" for MOUNTPOINT in "${SPLIT_MOUNTS[@]}"; do if [ "${MOUNTPOINT}" != "/tmp/in_target.d" ] && [ "${MOUNTPOINT}" != "/dev" ]; then sudo ${_runcon} chroot ${TARGET_ROOT} \ /usr/sbin/setfiles -F ${_dash_m} \ /etc/selinux/targeted/contexts/files/file_contexts ${MOUNTPOINT} fi done diskimage-builder-2.11.0/diskimage_builder/elements/rpm-distro/post-install.d/000077500000000000000000000000001324273057700274205ustar00rootroot0000000000000006-network-config-nonzeroconf000077500000000000000000000005471324273057700350120ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/rpm-distro/post-install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # zeroconf should not be activated or it will add a 169.254.0.0 # route. The route will interfere with access to the nova metadata # server at 169.254.169.254. if ! grep NOZEROCONF /etc/sysconfig/network ; then echo "NOZEROCONF=yes" >> /etc/sysconfig/network fi diskimage-builder-2.11.0/diskimage_builder/elements/rpm-distro/pre-install.d/000077500000000000000000000000001324273057700272215ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/rpm-distro/pre-install.d/00-fix-requiretty000077500000000000000000000007341324273057700323710ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # Fedora sets requiretty by default, which while great for machines that are # going to see a lot of passwords sent over the wire because of interactive # activity, that's not how we're using these. Remove the setting, and the # comments about it sed -i -e '/^Defaults\s*requiretty/d' -e '/Disable "ssh hostname/d' /etc/sudoers sed -i -e '/You have to run/d' /etc/sudoers visudo -c 00-usr-local-bin-secure-path000077500000000000000000000002631324273057700341730ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/rpm-distro/pre-install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail sed -i -e "/secure_path/ s@\([\'\"]\?\)\$@:/usr/local/bin/\1@" /etc/sudoers visudo -c diskimage-builder-2.11.0/diskimage_builder/elements/rpm-distro/pre-install.d/01-override-yum-arch000077500000000000000000000015021324273057700327250ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail if [ "i386" = "$ARCH" ]; then basearch=i386 arch=i686 elif [[ "amd64 x86_64" =~ "$ARCH" ]]; then basearch=x86_64 arch=x86_64 elif [[ "$ARCH" = "ppc64" ]]; then basearch=ppc64 arch=ppc64 elif [[ "$ARCH" = "ppc64le" ]]; then basearch=ppc64le arch=ppc64le elif [[ "arm64 aarch64" =~ "$ARCH" ]]; then basearch=aarch64 arch=aarch64 else echo "********************" echo "Unknown arch '$ARCH'" echo "********************" exit 1 fi if [[ $DISTRO_NAME == "fedora" && $DIB_RELEASE -ge 22 ]]; then mkdir -p /etc/dnf/vars echo $basearch > /etc/dnf/vars/basearch echo $arch > /etc/dnf/vars/arch else echo $basearch > /etc/yum/vars/basearch echo $arch > /etc/yum/vars/arch fi diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/000077500000000000000000000000001324273057700266475ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/README.rst000066400000000000000000000006341324273057700303410ustar00rootroot00000000000000===================== runtime-ssh-host-keys ===================== An element to generate SSH host keys on first boot. Since ssh key generation is not yet common to all operating systems, we need to create a DIB element to manage this. We force the removal of the SSH host keys, then add init scripts to generate them on first boot. This element currently supports Debian and Ubuntu (both systemd and upstart). diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/cleanup.d/000077500000000000000000000000001324273057700305205ustar00rootroot0000000000000090-remove-ssh-host-keys000077500000000000000000000005001324273057700346240ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/cleanup.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # Cloud images shouldn't have ssh host keys baked # in so that they are regenerated on first boot and # are unique. if [ -d $TARGET_ROOT/etc/ssh ] ; then sudo find $TARGET_ROOT/etc/ssh -name 'ssh_host*' -type f -delete fi diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/element-deps000066400000000000000000000000371324273057700311540ustar00rootroot00000000000000dib-init-system install-static diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/init-scripts/000077500000000000000000000000001324273057700312775ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/init-scripts/systemd/000077500000000000000000000000001324273057700327675ustar00rootroot00000000000000ssh-keygen.service000066400000000000000000000003111324273057700363420ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/init-scripts/systemd[Unit] Description=OpenSSH Server Key Generation Before=ssh.service [Service] ExecStart=/usr/local/sbin/runtime-ssh-host-keys.sh Type=oneshot RemainAfterExit=yes [Install] WantedBy=multi-user.target diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/init-scripts/upstart/000077500000000000000000000000001324273057700330015ustar00rootroot00000000000000ssh-keygen.conf000066400000000000000000000002071324273057700356450ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/init-scripts/upstartdescription "OpenSSH Server Key Generation" start on starting ssh console output task exec /usr/local/sbin/runtime-ssh-host-keys.sh diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/package-installs.yaml000066400000000000000000000000201324273057700327450ustar00rootroot00000000000000openssh-client: diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/pkg-map000066400000000000000000000002671324273057700301330ustar00rootroot00000000000000{ "family": { "redhat": { "openssh-client": "openssh" }, "gentoo": { "openssh-client": "" }, "suse": { "openssh-client": "openssh" } } } diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/post-install.d/000077500000000000000000000000001324273057700315225ustar00rootroot0000000000000080-ssh-keygen000077500000000000000000000014411324273057700336730ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/post-install.d#!/bin/bash if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail case "$DIB_INIT_SYSTEM" in upstart) # nothing to do exit 0 ;; systemd) if [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then # NOTE(pabelanger): Only support ubuntu / debian today. systemctl enable ssh-keygen.service else # Since we are not enabling it, delete it. rm /usr/lib/systemd/system/ssh-keygen.service fi ;; openrc) # let dib-init-system's postinstall handle enabling init scripts exit 0 ;; sysv) # nothing to do exit 0 ;; *) echo "Unsupported init system $DIB_INIT_SYSTEM" exit 1 ;; esac diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/static/000077500000000000000000000000001324273057700301365ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/static/usr/000077500000000000000000000000001324273057700307475ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/static/usr/local/000077500000000000000000000000001324273057700320415ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/static/usr/local/sbin/000077500000000000000000000000001324273057700327745ustar00rootroot00000000000000runtime-ssh-host-keys.sh000077500000000000000000000020231324273057700374530ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/runtime-ssh-host-keys/static/usr/local/sbin#!/bin/bash # Copyright 2016 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # dib-lint: disable=dibdebugtrace setpipefail set -exu # We are running into race conditions with glean, which ssh-keygen -A is # not handling properly. So, create a new script to first check if the # file exists, then use 'yes' to disable overwriting of existing files. for key in dsa ecdsa ed25519 rsa; do FILE=/etc/ssh/ssh_host_${key}_key if ! [ -f $FILE ]; then /usr/bin/yes n | /usr/bin/ssh-keygen -f $FILE -N '' -t $key fi done diskimage-builder-2.11.0/diskimage_builder/elements/select-boot-kernel-initrd/000077500000000000000000000000001324273057700274325ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/select-boot-kernel-initrd/README.rst000066400000000000000000000005271324273057700311250ustar00rootroot00000000000000========================= select-boot-kernel-initrd ========================= A helper script to get the kernel and initrd image. It uses the function select_boot_kernel_initrd from the library img-functions to find the newest kernel and ramdisk in the image, and returns them as a concatenated string separating the values with a colon (:). diskimage-builder-2.11.0/diskimage_builder/elements/select-boot-kernel-initrd/bin/000077500000000000000000000000001324273057700302025ustar00rootroot00000000000000select-boot-kernel-initrd000077500000000000000000000014761324273057700350460ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/select-boot-kernel-initrd/bin#!/bin/bash # Copyright 2012 Hewlett-Packard Development Company, L.P. # Copyright 2013 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail source /lib/diskimage-builder/img-functions select_boot_kernel_initrd / echo ${KERNEL}:${RAMDISK} diskimage-builder-2.11.0/diskimage_builder/elements/select-boot-kernel-initrd/cleanup.d/000077500000000000000000000000001324273057700313035ustar00rootroot0000000000000099-remove-dib-img-functions000077500000000000000000000003151324273057700362210ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/select-boot-kernel-initrd/cleanup.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail sudo rm -f $TMP_MOUNT_PATH/lib/diskimage-builder/img-functions sudo rmdir $TMP_MOUNT_PATH/lib/diskimage-builder diskimage-builder-2.11.0/diskimage_builder/elements/select-boot-kernel-initrd/extra-data.d/000077500000000000000000000000001324273057700317065ustar00rootroot0000000000000099-copy-dib-img-functions000077500000000000000000000004531324273057700363040ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/select-boot-kernel-initrd/extra-data.d#!/bin/bash # Copy elements-functions into chroot so we can access the get_kernel_image function if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail sudo mkdir -p $TMP_MOUNT_PATH/lib/diskimage-builder sudo cp -t $TMP_MOUNT_PATH/lib/diskimage-builder $_LIB/img-functions diskimage-builder-2.11.0/diskimage_builder/elements/selinux-permissive/000077500000000000000000000000001324273057700263205ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/selinux-permissive/README.rst000066400000000000000000000003041324273057700300040ustar00rootroot00000000000000================== selinux-permissive ================== Puts selinux into permissive mode by writing SELINUX=permissive to /etc/selinux/config Enable this element when debugging selinux issues. diskimage-builder-2.11.0/diskimage_builder/elements/selinux-permissive/install.d/000077500000000000000000000000001324273057700302105ustar00rootroot0000000000000011-selinux-permissive000077500000000000000000000003011324273057700341430ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/selinux-permissive/install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail echo "SELINUX=permissive" > /etc/selinux/config echo "SELINUXTYPE=targeted" >> /etc/selinux/config diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/000077500000000000000000000000001324273057700246775ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/README.rst000066400000000000000000000045371324273057700263770ustar00rootroot00000000000000=========== simple-init =========== Basic network and system configuration that can't be done until boot Unfortunately, as much as we'd like to bake it in to an image, we can't know in advance how many network devices will be present, nor if DHCP is present in the host cloud. Additionally, in environments where cloud-init is not used, there are a couple of small things, like mounting config-drive and pulling ssh keys from it, that need to be done at boot time. Autodetect network interfaces during boot and configure them ------------------------------------------------------------ The rationale for this is that we are likely to require multiple network interfaces for use cases such as baremetal and there is no way to know ahead of time which one is which, so we will simply run a DHCP client on all interfaces with real MAC addresses (except lo) that are visible on the first boot. The script `/usr/local/sbin/simple-init.sh` will be called early in each boot and will scan available network interfaces and ensure they are configured properly before networking services are started. Processing startup information from config-drive ------------------------------------------------ On most systems, the DHCP approach desribed above is fine. But in some clouds, such as Rackspace Public cloud, there is no DHCP. Instead, there is static network config via `config-drive`. `simple-init` will happily call `glean` which will do nothing if static network information is not there. Finally, glean will handle ssh-keypair-injection from config drive if cloud-init is not installed. Chosing glean installation source --------------------------------- By default glean is installed using pip using the latest release on pypi. It is also possible to install glean from a specified git repository location. This is useful for debugging and testing new glean changes for example. To do this you need to set these variables:: DIB_INSTALLTYPE_simple_init=repo DIB_REPOLOCATION_glean=/path/to/glean/repo DIB_REPOREF_glean=name_of_git_ref For example to test glean change 364516 do:: git clone https://git.openstack.org/openstack-infra/glean /tmp/glean cd /tmp/glean git review -d 364516 git checkout -b my-test-ref Then set your DIB env vars like this before running DIB:: DIB_INSTALLTYPE_simple_init=repo DIB_REPOLOCATION_glean=/tmp/glean DIB_REPOREF_glean=my-test-ref diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/element-deps000066400000000000000000000001621324273057700272030ustar00rootroot00000000000000cloud-init-datasources dib-init-system install-types pip-and-virtualenv runtime-ssh-host-keys source-repositories diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/environment.d/000077500000000000000000000000001324273057700274655ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/environment.d/50-disable-cloud-init000066400000000000000000000000511324273057700332760ustar00rootroot00000000000000export DIB_CLOUD_INIT_DATASOURCES="None" diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/install.d/000077500000000000000000000000001324273057700265675ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/install.d/50-simple-init000077500000000000000000000014251324273057700311730ustar00rootroot00000000000000#!/bin/bash # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # # See the License for the specific language governing permissions and # limitations under the License. if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail SCRIPTDIR=$(dirname $0) # Ensure the 8021q module is loaded so that glean can support tagged interfaces echo "8021q" >> /etc/modules glean-install 60-simple-init-remove-interfaces000077500000000000000000000012701324273057700345270ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # Cloud images may hard code the eth0 interfaces so they # boot with DHCP. # Fedora rm -f /etc/sysconfig/network-scripts/ifcfg-eth* # SUSE rm -f /etc/sysconfig/network/ifcfg-eth* # Ubuntu rm -f /etc/network/interfaces.d/eth* # Debian rm -f /etc/network/interfaces.d/eth* # Gentoo rm -f /etc/conf.d/net* # /etc/network/interfaces distributions if [ -f "/etc/network/interfaces" ]; then printf "auto lo\niface lo inet loopback\n\n" > /etc/network/interfaces if [ -d "/etc/network/interfaces.d/" ]; then printf "source-directory interfaces.d\n\n" >> /etc/network/interfaces fi fi diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/install.d/simple-init-repo-install/000077500000000000000000000000001324273057700334305ustar00rootroot0000000000000040-glean000077500000000000000000000013341324273057700346070ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/install.d/simple-init-repo-install#!/bin/bash # Copyright (c) 2015 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # # See the License for the specific language governing permissions and # limitations under the License. if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail pip install /tmp/glean.git simple-init-source-install/000077500000000000000000000000001324273057700337045ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/install.d40-glean000077500000000000000000000013231324273057700351400ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/install.d/simple-init-source-install#!/bin/bash # Copyright (c) 2015 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # # See the License for the specific language governing permissions and # limitations under the License. if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail pip install glean diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/package-installs.yaml000066400000000000000000000000421324273057700310010ustar00rootroot00000000000000dhcp-client: net-tools: ifupdown: diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/pkg-map000066400000000000000000000006701324273057700261610ustar00rootroot00000000000000{ "family": { "redhat": { "dhcp-client": "dhclient" }, "debian": { "dhcp-client": "isc-dhcp-client", "ifupdown": "ifupdown" }, "gentoo": { "dhcp-client": "net-misc/dhcp", "python-pip": "dev-python/pip", "net-tools": "sys-apps/net-tools" }, "suse": { "dhcp-client": "dhcp-client" } }, "default": { "dhcp-client": "isc-dhcp-client", "ifupdown": "" } } diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/post-install.d/000077500000000000000000000000001324273057700275525ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/post-install.d/80-simple-init000077500000000000000000000015761324273057700321700ustar00rootroot00000000000000#!/bin/bash if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail case "$DIB_INIT_SYSTEM" in upstart) # nothing to do exit 0 ;; systemd) if [[ ${DISTRO_NAME} =~ (centos|rhel7|fedora) ]]; then # NOTE(pabelanger): Glean requires network.service for # these platforms. systemctl enable network.service elif [[ ${DISTRO_NAME} =~ (opensuse) ]]; then # on suse, this is named wicked.service, but it's the same # as network.service. systemctl enable wicked.service fi ;; openrc) # let dib-init-system's postinstall handle enabling init scripts exit 0 ;; sysv) # nothing to do exit 0 ;; *) echo "Unsupported init system $DIB_INIT_SYSTEM" exit 1 ;; esac diskimage-builder-2.11.0/diskimage_builder/elements/simple-init/source-repository-simple-init000066400000000000000000000001111324273057700325600ustar00rootroot00000000000000glean git /tmp/glean.git https://git.openstack.org/openstack-infra/glean diskimage-builder-2.11.0/diskimage_builder/elements/source-repositories/000077500000000000000000000000001324273057700264725ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/source-repositories/README.rst000066400000000000000000000134021324273057700301610ustar00rootroot00000000000000=================== source-repositories =================== With this element other elements can register their installation source by placing their details in the file ``source-repository-*``. source-repository-* file format ------------------------------- The plain text file format is space separated and has four mandatory fields optionally followed by fields which are type dependent:: [] ``name`` Identifier for the source repository. Should match the file suffix. ``type`` Format of the source. Either ``git``, ``tar``, ``package`` or ``file``. ``destination`` Base path to place sources. ``location`` Resource to fetch sources from. For ``git`` the location is cloned. For ``tar`` it is extracted. ``ref`` (optional). Meaning depends on the ``type``: ``file``: unused/ignored. ``git``: a git reference to fetch. A value of "``*``" prunes and fetches all heads and tags. Defaults to ``master`` if not specified. ``tar``: | "``.``" extracts the entire contents of the tarball. | "``*``" extracts the contents within all its subdirectories. | A subdirectory path may be used to extract only its contents. | A specific file path within the archive is **not supported**. The lines in the source-repository scripts are eval'd, so they may contain environment variables. The ``package`` type indicates the element should install from packages onto the root filesystem of the image build during the ``install.d`` phase. If the element provides an -package-install directory, symlinks will be created for those scripts instead. ``git`` and ``tar`` are treated as source installs. If the element provides an -source-install directory under it's ``install.d`` hook directory, symlinks to the scripts in that directory will be created under ``install.d`` for the image build. For example, the nova element would provide:: nova/install.d/nova-package-install/74-nova nova/install.d/nova-source-install/74-nova source-repositories will create the following symlink for the package install type:: install.d/74-nova -> nova-package-install/74-nova Or, for the source install type:: install.d/74-nova -> nova-source-install/74-nova All other scripts that exist under ``install.d`` for an element will be executed as normal. This allows common install code to live in a script outside of -package-install or -source-install. If multiple elements register a source location with the same then source-repositories will exit with an error. Care should therefore be taken to only use elements together that download source to different locations. The repository paths built into the image are stored in etc/dib-source-repositories, one repository per line. This permits later review of the repositories (by users or by other elements). The repository names and types are written to an environment.d hook script at 01-source-repositories-environment. This allows later hook scripts during the install.d phase to know which install type to use for the element. An example of an element "custom-element" that wants to retrieve the ironic source from git and pbr from a tarball would be: *Element file: elements/custom-element/source-repository-ironic*:: ironic git /usr/local/ironic git://git.openstack.org/openstack/ironic.git *File : elements/custom-element/source-repository-pbr*:: pbr tar /usr/local/pbr http://tarballs.openstack.org/pbr/pbr-master.tar.gz . diskimage-builder will then retrieve the sources specified and place them at the directory ````. Override per source ------------------- A number of environment variables can be set by the process calling diskimage-builder which can change the details registered by the element, these are: ``DIB_REPOTYPE_`` : change the registered type ``DIB_REPOLOCATION_`` : change the registered location ``DIB_REPOREF_`` : change the registered reference For example if you would like diskimage-builder to get ironic from a local mirror you would override the ```` field and could set: .. sourcecode:: sh DIB_REPOLOCATION_ironic=git://localgitserver/ironic.git *As you can see above, the \ of the repo is used in several bash variables. In order to make this syntactically feasible, any characters not in the set \[A-Za-z0-9_\] will be converted to \_* *For instance, a repository named "diskimage-builder" would set a variable called "DIB_REPOTYPE_diskimage_builder"* Alternatively if you would like to use the keystone element and build an image with keystone from a stable branch *stable/grizzly* then you would set: .. sourcecode:: sh DIB_REPOREF_keystone=stable/grizzly If you wish to build an image using code from a Gerrit review, you can set ``DIB_REPOLOCATION_`` and ``DIB_REPOREF_`` to the values given by Gerrit in the fetch/pull section of a review. For example, setting up Nova with change 61972 at patchset 8: .. sourcecode:: sh DIB_REPOLOCATION_nova=https://review.openstack.org/openstack/nova DIB_REPOREF_nova=refs/changes/72/61972/8 Alternate behaviors ------------------- Override git remote ^^^^^^^^^^^^^^^^^^^ The base url for all git repositories can be set by use of: ``DIB_GITREPOBASE`` So setting ``DIB_GITREPOBASE=https://github.com/`` when the repo location is set to http://git.openstack.org/openstack/nova.git will result in use of the https://github.com/openstack/nova.git repository instead. Disable external fetches ^^^^^^^^^^^^^^^^^^^^^^^^ When doing image builds in environments where external resources are not allowed, it is possible to disable fetching of all source repositories by including an element in the image that sets ``NO_SOURCE_REPOSITORIES=1`` in an ``environment.d`` script. diskimage-builder-2.11.0/diskimage_builder/elements/source-repositories/element-deps000066400000000000000000000000451324273057700307760ustar00rootroot00000000000000cache-url manifests package-installs diskimage-builder-2.11.0/diskimage_builder/elements/source-repositories/extra-data.d/000077500000000000000000000000001324273057700307465ustar00rootroot0000000000000098-source-repositories000077500000000000000000000301761324273057700351070ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/source-repositories/extra-data.d#!/bin/bash # dib-lint: disable=safe_sudo if [ ${DIB_DEBUG_TRACE:-0} -gt 1 ]; then set -x fi set -eu set -o pipefail # Don't provide any source repositories in environments where they are not allowed if [ -n "${NO_SOURCE_REPOSITORIES:-}" ]; then exit 0 fi # If the old cache exists, move it to the new name function make_new_cache(){ local OLD_CACHE_BASE=$1 local CACHE_BASE=$2 # If the old cache name exists, move it to the new cache name if [ -e "$OLD_CACHE_BASE" ] ; then if [ ! -e "$CACHE_BASE" ] ; then mv -n $OLD_CACHE_BASE $CACHE_BASE else echo "Not replacing new cache location with old cache" fi fi } # Gets repositories or individual files listed in the a repository file # and places them in the specified destination path. # The format of the repository file is one or more lines matching # [] function get_repos_for_element(){ local REPO_SOURCES=$1 local CACHE_URL=$TMP_HOOKS_PATH/bin/cache-url local REGEX="^([^ ]+) (git|tar|file|package) ?(/[^ ]+)? ?([^ ]+)? ?([^ ]*)$" # this can be a rather long list (e.g. caching ~1000 openstack # repos), so we make a rough count for progress display local COUNT=1 local REPO_SOURCES_COUNT=$(grep -v '^$\|^\s*\#' $REPO_SOURCES | wc -l) while read line; do # temporarily turn off globbing '*' (since it may be used as # the REPOREF for tarballs) and expand variables set -f; line=$(eval echo $line); set +f # ignore blank lines and lines beginning in '#' [[ "$line" == \#* ]] || [[ -z "$line" ]] && continue # rough progress counter printf "(%04d / %04d)\n" ${COUNT} ${REPO_SOURCES_COUNT} COUNT=$(( COUNT + 1)) if [[ "$line" =~ $REGEX ]] ; then local REPONAME=${BASH_REMATCH[1]} local REPOTYPE=${BASH_REMATCH[2]} local REPOPATH=${BASH_REMATCH[3]} local REPOLOCATION=${BASH_REMATCH[4]} local REPO_ORIG_LOCATION=$REPOLOCATION if [ $REPONAME = "tar" -a -z "${BASH_REMATCH[5]:-}" ] ; then echo "Warning: Default tarball REPOREF of '*' is deprecated; do not rely on it." fi # Default of '*' for tar repositories is deprecated; do not rely on it. local REPOREF_DEFAULT_TAR=* local REPOREF_DEFAULT_GIT=master local REPOREF_LOOKUP_DEFAULT=REPOREF_DEFAULT_${REPOTYPE^^} local REPOREF=${BASH_REMATCH[5]:-${!REPOREF_LOOKUP_DEFAULT:-}} local REPO_DEST=$TMP_MOUNT_PATH$REPOPATH local REPO_SUB_DIRECTORY=$(dirname $REPO_DEST) # REPOTYPE can be overridden with DIB_REPOTYPE_{name} local REPOTYPE_OVERRIDE=DIB_REPOTYPE_${REPONAME//[^A-Za-z0-9]/_} REPOTYPE=${!REPOTYPE_OVERRIDE:-$REPOTYPE} # REPOLOCATION can be overridden with DIB_REPOLOCATION_{name} local REPOLOCATION_OVERRIDE=DIB_REPOLOCATION_${REPONAME//[^A-Za-z0-9]/_} REPOLOCATION=${!REPOLOCATION_OVERRIDE:-$REPOLOCATION} # REPOREF can be overridden with DIB_REPOREF_{name} local REPOREF_OVERRIDE=DIB_REPOREF_${REPONAME//[^A-Za-z0-9]/_} REPOREF=${!REPOREF_OVERRIDE:-$REPOREF} # Determine a unique cache path for this repo CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' ) OLD_CACHE_PATH=${CACHE_BASE}/${CACHE_NAME} # Add the repo name to the sha1sum for readability CACHE_NAME=${REPONAME//[^A-Za-z0-9]/_}_${CACHE_NAME} CACHE_PATH=${CACHE_BASE}/$CACHE_NAME make_new_cache $OLD_CACHE_PATH $CACHE_PATH # Return if install type is not source local INSTALL_TYPE_VAR=DIB_INSTALLTYPE_${REPONAME//[^A-Za-z0-9]/_} local INSTALL_TYPE=${!INSTALL_TYPE_VAR:-$DIB_DEFAULT_INSTALLTYPE} if [ ! $INSTALL_TYPE = "source" ]; then echo "$REPONAME install type not set to source" continue fi case $REPOTYPE in git) if [ -z "${!REPOLOCATION_OVERRIDE:-""}" -a -n "${DIB_GITREPOBASE:-""}" ] ; then # Transform the current repo base to the new one local NEW_REPOLOCATION=$(echo $REPOLOCATION |\ sed "s,^[^:]\+://[^/]\+/\(~[^/]\+\)\?\(.*\)$,${DIB_GITREPOBASE}\2,g") echo "Transformed ${REPOLOCATION} to ${NEW_REPOLOCATION}" REPOLOCATION=$NEW_REPOLOCATION # Also update the cache location CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' ) CACHE_PATH=~/.cache/image-create/repository-sources/$CACHE_NAME fi sudo mkdir -p $REPO_SUB_DIRECTORY if [ ! -e "$CACHE_PATH" ] ; then echo "Caching $REPONAME from $REPOLOCATION in $CACHE_PATH" git clone -q $REPOLOCATION $CACHE_PATH.tmp mv ${CACHE_PATH}{.tmp,} fi if [ "$REPOREF" != "*" ] ; then HAS_REF=$(git --git-dir=$CACHE_PATH/.git name-rev $REPOREF 2>/dev/null || true) fi if [ -z "$DIB_OFFLINE" -o -z "${HAS_REF:-}" ] ; then echo "Updating cache of $REPOLOCATION in $CACHE_PATH with ref $REPOREF" # Copy named refs (which might be outside the usual heads # pattern) - e.g. gerrit if [ "$REPOREF" == "*" ] || ! git --git-dir=$CACHE_PATH/.git fetch -q --prune --update-head-ok $REPOLOCATION \ +${REPOREF}:${REPOREF} ; then # Copy all heads from the remote repository - this permits # using a SHA1 object reference so long as the object # reference is reachable from one of the heads. git does # not permit arbitrary sha fetching from remote servers. # This is a separate fetch to the prior one as the prior # one will fail when REPOREF is a SHA1. git --git-dir=$CACHE_PATH/.git fetch -q --prune --update-head-ok $REPOLOCATION \ +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/* fi # Ensure that we have a reference to the revision. if [ "$REPOREF" != "*" ] ; then if ! git --git-dir=$CACHE_PATH/.git rev-parse -q --verify $REPOREF^{commit} > /dev/null; then echo "Failed to find reference to $REPOREF" exit 1 fi fi fi echo "Cloning from $REPONAME cache and applying ref $REPOREF" # If the local dir is already used, see if the pertinent details differ if [[ -d $REPO_DEST ]]; then DESIRED="$(sudo git --git-dir=$REPO_DEST/.git config remote.origin.url)" if [[ "$CACHE_PATH" != "$DESIRED" ]]; then echo "REPOLOCATIONS don't match ("$CACHE_PATH" != "$DESIRED")" >&2 exit 1 elif [[ "$REPOREF" != "*" ]]; then pushd $REPO_DEST > /dev/null # When we first clone we create a branch naming what we fetched # that must match, or we are asking for two different references from the # same repo, which is an error if ! git rev-parse fetch_$REPOREF; then echo "REPOREFS don't match - failed to get sha1 of fetch_$REPOREF" >&2 exit 1 fi popd > /dev/null fi else sudo git clone -q $CACHE_PATH $REPO_DEST pushd $REPO_DEST > /dev/null if [[ "$REPOREF" == "*" ]]; then sudo git fetch -q --prune --update-head-ok $CACHE_PATH \ +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/* git_sha=$(git rev-parse HEAD) else sudo git fetch -q $CACHE_PATH $REPOREF:fetch_$REPOREF sudo git reset --hard FETCH_HEAD # Get the sha in use git_sha=$(git rev-parse FETCH_HEAD) fi popd > /dev/null # Write the sha being used into the source-repositories manifest echo "$REPONAME git $REPOPATH $REPOLOCATION $git_sha" >> $GIT_MANIFEST fi ;; tar) # The top level directory of the tarball mightn't have a fixed name i.e. # it could contain version numbers etc... so we write it to a tmpdir # for inspection before transferring the contents into the target directory local tmpdir=$(mktemp --tmpdir=$TMP_MOUNT_PATH/tmp -d) if [ -n "$CACHE_PATH" ] ; then echo "Caching $REPONAME tarball from $REPOLOCATION in $CACHE_PATH" if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then $CACHE_URL $REPOLOCATION $CACHE_PATH fi tar -C $tmpdir -xzf $CACHE_PATH else echo "Fetching $REPONAME tarball from $REPOLOCATION" curl $REPOLOCATION | tar -C $tmpdir -xzf - fi sudo mkdir -p $REPO_DEST # A REPOREF of '.' will select the entire contents of the tarball, # while '*' will select only the contents of its subdirectories. sudo rsync -a --remove-source-files $tmpdir/$REPOREF/. $REPO_DEST rm -rf $tmpdir ;; file) sudo mkdir -p $REPO_SUB_DIRECTORY if [ -n "$CACHE_PATH" ] ; then echo "Caching $REPONAME file from $REPOLOCATION in $CACHE_PATH" if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then $CACHE_URL $REPOLOCATION $CACHE_PATH fi sudo cp $CACHE_PATH $REPO_DEST else echo "Fetching $REPONAME file from $REPOLOCATION" sudo curl $REPOLOCATION -o $REPO_DEST fi ;; *) echo "Unsupported repository type: $REPOTYPE" return 1 ;; esac # Capture the in-instance repository path for later review / other # elements (like a pypi dependency cache). echo "$REPOPATH" | sudo tee -a $TMP_MOUNT_PATH/etc/dib-source-repositories > /dev/null else echo "Couldn't parse '$line' as a source repository" return 1 fi done < $REPO_SOURCES } CACHE_BASE=$DIB_IMAGE_CACHE/source-repositories OLD_CACHE_BASE=$DIB_IMAGE_CACHE/repository-sources make_new_cache $OLD_CACHE_BASE $CACHE_BASE mkdir -p $CACHE_BASE # Use the IMAGE_NAME from the calling script, and make it unique with the temporary path GIT_MANIFEST_NAME=dib-manifest-git-$(basename ${IMAGE_NAME}) GIT_MANIFEST_CACHE_NAME=${GIT_MANIFEST_NAME}_$(dirname ${TMP_MOUNT_PATH##*.}) GIT_MANIFEST=$CACHE_BASE/${GIT_MANIFEST_CACHE_NAME} rm -f $GIT_MANIFEST # Get source repositories for the target for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "source-repository-*" -not -name '*~'); do repo_flock=$CACHE_BASE/repositories_flock ( echo "Getting $repo_flock: $(date) for $_SOURCEREPO" if ! flock -w 600 9 ; then echo "Did not get $repo_flock: $(date)" exit 1 fi get_repos_for_element $_SOURCEREPO ) 9> $repo_flock done # Copy the manifest into the image if it exists (there may be no git repositories used) if [ -e "$GIT_MANIFEST" ] ; then sudo mv $GIT_MANIFEST $TMP_MOUNT_PATH/${DIB_MANIFEST_IMAGE_DIR}/$GIT_MANIFEST_NAME fi diskimage-builder-2.11.0/diskimage_builder/elements/source-repositories/package-installs.yaml000066400000000000000000000003501324273057700325760ustar00rootroot00000000000000# Git isn't a dependency of this element, but is a dependency of pbr # if you are doing pip install on a git cloned directory, because this # looks to be a fairly common pattern it makes sense to have git on the # target image git: diskimage-builder-2.11.0/diskimage_builder/elements/source-repositories/pkg-map000066400000000000000000000002311324273057700277450ustar00rootroot00000000000000{ "family": { "gentoo": { "git": "dev-vcs/git" }, "suse": { "git": "git-core" } }, "default": { "git": "git" } } diskimage-builder-2.11.0/diskimage_builder/elements/stable-interface-names/000077500000000000000000000000001324273057700267565ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/stable-interface-names/README.rst000066400000000000000000000003651324273057700304510ustar00rootroot00000000000000====================== stable-interface-names ====================== Does the following: * Enables stable network interface naming (em1, em2, etc) by installing the biosdevname and removes any symlinks which may disable udev rules, etc. diskimage-builder-2.11.0/diskimage_builder/elements/stable-interface-names/element-deps000066400000000000000000000000211324273057700312540ustar00rootroot00000000000000package-installs diskimage-builder-2.11.0/diskimage_builder/elements/stable-interface-names/install.d/000077500000000000000000000000001324273057700306465ustar00rootroot0000000000000002-stable-interface-names000077500000000000000000000004271324273057700352500ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/stable-interface-names/install.d#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # remove softlink to /dev/null which disables these rules NET_NAME_SLOT_FILE="/etc/udev/rules.d/80-net-name-slot.rules" if [ -h $NET_NAME_SLOT_FILE ]; then rm $NET_NAME_SLOT_FILE fi diskimage-builder-2.11.0/diskimage_builder/elements/stable-interface-names/package-installs.yaml000066400000000000000000000000151324273057700330600ustar00rootroot00000000000000biosdevname: diskimage-builder-2.11.0/diskimage_builder/elements/svc-map/000077500000000000000000000000001324273057700240135ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/svc-map/README.rst000066400000000000000000000060641324273057700255100ustar00rootroot00000000000000======= svc-map ======= Map service names to distro specific services. Provides the following: * bin/svc-map usage: svc-map [-h] SERVICE Translate service name to distro specific name. optional arguments: -h, --help show this help message and exit * Any element may create its own svc-map YAML config file using the one of 3 sections for the distro/family/ and or default. The family is set automatically within svc-map based on the supplied distro name. Families include: + redhat: includes centos, fedora, and rhel distros + debian: includes debian and ubuntu distros + suse: includes the opensuse distro The most specific section takes priority. Example for Nova and Glance (NOTE: default is using the common value for redhat and suse families) The key used for the service name should always be the same name used for the source installation of the service. The svc-map script will check for the source name against systemd and upstart and return that name if it exists instead of the mapped name. Example format for Nova:: nova-api: default: openstack-nova-api debian: nova-api nova-cert: default: openstack-nova-cert debian: nova-cert nova-compute: default: openstack-nova-compute debian: nova-compute nova-conductor: default: openstack-nova-conductor debian: nova-conductor nova-consoleauth: default: openstack-nova-console debian: nova-console Example format for Glance:: glance-api: debian: glance-api default: openstack-glance-api glance-reg: debian: glance-reg default: openstack-glance-registry If the distro is of the debian family the combined services file would be:: nova-cert: nova-cert nova-compute: nova-compute glance-api: glance-api nova-conductor: nova-conductor nova-api: nova-api glance-reg: glance-reg nova-consoleauth: nova-console If the distro is of the suse or redhat families the combined services file would be:: nova-cert: openstack-nova-cert nova-compute: openstack-nova-compute glance-reg: openstack-glance-registry nova-conductor: openstack-nova-conductor glance-api: openstack-glance-api nova-consoleauth: openstack-nova-console nova-api: openstack-nova-api Example commands using this format:: svc-map nova-compute Returns: openstack-nova-compute svc-map nova-compute Returns: openstack-nova-compute svc-map nova-compute Returns: nova-compute * This output can be used to filter what other tools actually install (install-services can be modified to use this for example) * If you pass more than one service argument, the result for each service is printed on its own line. * Individual svc-map files live within each element. For example if you have created an Apache element your svc-map YAML file should be created at elements/apache/svc-map. diskimage-builder-2.11.0/diskimage_builder/elements/svc-map/bin/000077500000000000000000000000001324273057700245635ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/svc-map/bin/svc-map000077500000000000000000000027561324273057700260710ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2012 Hewlett-Packard Development Company, L.P. # Copyright 2014 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import os import sys import yaml def load_service_mapping(filepath="/usr/share/svc-map/services"): if not os.path.isfile(filepath): return {} with open(filepath, 'r') as data_file: return yaml.safe_load(data_file.read()) def main(): for arg in sys.argv[1:]: # We need to support the service name being different when installing # from source vs. packages. So, if the requested service file already # exists, just use that. if os.path.exists('/lib/systemd/system/%s.service' % arg): # systemd print(arg) elif os.path.exists('/etc/init/%s.conf' % arg): # upstart print(arg) else: service_map = load_service_mapping() name = service_map.get(arg, arg) print(name) sys.exit(0) if __name__ == '__main__': main() diskimage-builder-2.11.0/diskimage_builder/elements/svc-map/extra-data.d/000077500000000000000000000000001324273057700262675ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/svc-map/extra-data.d/10-merge-svc-map-files000077500000000000000000000054141324273057700322020ustar00rootroot00000000000000#!/usr/bin/env python # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import os import sys import yaml def os_family(distro): "Given a distribution, returns a operating system family." family = None if distro in ['centos', 'centos7', 'fedora', 'rhel', 'rhel7']: family = 'redhat' elif distro in ['debian', 'ubuntu']: family = 'debian' elif distro == 'opensuse': family = 'suse' return family def merge_data(source, destination, distro): """Merges two dictionaries and filters on distro. Merges two dictionaries and filters on distro family, or default (in order) """ result = dict() result.update(destination) family = os_family(distro) for servicename, mapping in source.items(): if servicename in result: raise Exception("%s already found in services list" % servicename) if distro in mapping: result[servicename] = mapping.get(distro) elif family in mapping: result[servicename] = mapping.get(family) elif 'default' in mapping: result[servicename] = mapping.get('default') else: result[servicename] = servicename return result def write_data_to_file(data, service_file_path): """Writes yaml data to a specified path.""" with open(service_file_path, 'w') as destination: yaml.dump(data, destination, default_flow_style=False) def main(): elements = yaml.safe_load(os.environ.get("IMAGE_ELEMENT_YAML")) service_names = dict() for element, path in elements.items(): data_path = os.path.join(path, "svc-map") if os.path.exists(data_path): with open(data_path, 'r') as dataFile: data = yaml.safe_load(dataFile.read()) try: service_names = merge_data( data, service_names, os.environ.get("DISTRO_NAME")) except Exception as err: print("%s. Check %s for duplicate" " service name." % (err, element)) sys.exit(1) write_data_to_file( service_names, os.path.join(os.environ['TMP_MOUNT_PATH'], 'tmp', 'svc-map-services')) if __name__ == "__main__": main() diskimage-builder-2.11.0/diskimage_builder/elements/svc-map/extra-data.d/11-copy-svc-map-file000077500000000000000000000003131324273057700316640ustar00rootroot00000000000000#!/bin/bash if [ "${DIB_DEBUG_TRACE:-1}" -gt 0 ]; then set -x fi set -eu set -o pipefail sudo install -D -m 0644 "$TMP_MOUNT_PATH/tmp/svc-map-services" "$TMP_MOUNT_PATH/usr/share/svc-map/services" diskimage-builder-2.11.0/diskimage_builder/elements/svc-map/package-installs.yaml000066400000000000000000000000411324273057700301140ustar00rootroot00000000000000PyYAML: phase: pre-install.d diskimage-builder-2.11.0/diskimage_builder/elements/svc-map/pkg-map000066400000000000000000000003321324273057700252700ustar00rootroot00000000000000{ "family": { "redhat": { "PyYAML": "PyYAML" }, "debian": { "PyYAML": "python-yaml" }, "suse": { "PyYAML": "python-PyYAML" } }, "default": { "PyYAML": "PyYAML" } } diskimage-builder-2.11.0/diskimage_builder/elements/svc-map/tests/000077500000000000000000000000001324273057700251555ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/svc-map/tests/__init__.py000066400000000000000000000000001324273057700272540ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/svc-map/tests/test_data_merge.py000066400000000000000000000145161324273057700306650ustar00rootroot00000000000000# Copyright 2014 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import imp import os from oslotest import base module_path = (os.path.dirname(os.path.realpath(__file__)) + '/../extra-data.d/10-merge-svc-map-files') service_map = imp.load_source('service_map', module_path) class TestDataMerge(base.BaseTestCase): nova_api_services = { u'nova-api': { u'debian': u'nova-api', u'default': u'openstack-nova-api' }, u'nova-cert': { u'debian': u'nova-cert', u'default': u'openstack-nova-cert' }, u'nova-compute': { u'debian': u'nova-compute', u'default': u'openstack-nova-compute' }, u'nova-conductor': { u'debian': u'nova-conductor', u'default': u'openstack-nova-conductor' }, u'nova-consoleauth': { u'debian': u'nova-console', u'default': u'openstack-nova-console' } } glance_api_services = { u'glance-api': { u'debian': u'glance-api', u'default': u'openstack-glance-api' }, u'glance-reg': { u'debian': u'glance-reg', u'default': u'openstack-glance-registry' } } cinder_api_services = { u'cinder-api': { u'debian': u'cinder-api', u'default': u'openstack-cinder-api' }, u'cinder-scheduler': { u'debian': u'cinder-scheduler', u'default': u'openstack-cinder-scheduler' } } def test_merge_data_fedora(self): fedora_nova_api_services = { u'nova-api': u'openstack-nova-api', u'nova-cert': u'openstack-nova-cert', u'nova-compute': u'openstack-nova-compute', u'nova-conductor': u'openstack-nova-conductor', u'nova-consoleauth': u'openstack-nova-console' } fedora_nova_glance_services = { u'nova-api': u'openstack-nova-api', u'nova-cert': u'openstack-nova-cert', u'nova-compute': u'openstack-nova-compute', u'nova-conductor': u'openstack-nova-conductor', u'nova-consoleauth': u'openstack-nova-console', u'glance-api': u'openstack-glance-api', u'glance-reg': u'openstack-glance-registry' } fedora_nova_glance_cinder_services = { u'nova-api': u'openstack-nova-api', u'nova-cert': u'openstack-nova-cert', u'nova-compute': u'openstack-nova-compute', u'nova-conductor': u'openstack-nova-conductor', u'nova-consoleauth': u'openstack-nova-console', u'glance-api': u'openstack-glance-api', u'glance-reg': u'openstack-glance-registry', u'cinder-api': u'openstack-cinder-api', u'cinder-scheduler': u'openstack-cinder-scheduler', } result = dict() result = service_map.merge_data(self.nova_api_services, result, "fedora") self.assertDictEqual(result, fedora_nova_api_services, "Merge failed") result = service_map.merge_data(self.glance_api_services, result, "fedora") self.assertDictEqual(result, fedora_nova_glance_services, "Merge failed") result = service_map.merge_data(self.cinder_api_services, result, "fedora") self.assertDictEqual(result, fedora_nova_glance_cinder_services, "Merge failed") def test_merge_data_ubuntu(self): ubuntu_nova_api_services = { u'nova-api': u'nova-api', u'nova-cert': u'nova-cert', u'nova-compute': u'nova-compute', u'nova-conductor': u'nova-conductor', u'nova-consoleauth': u'nova-console' } ubuntu_nova_glance_services = { u'nova-api': u'nova-api', u'nova-cert': u'nova-cert', u'nova-compute': u'nova-compute', u'nova-conductor': u'nova-conductor', u'nova-consoleauth': u'nova-console', u'glance-api': u'glance-api', u'glance-reg': u'glance-reg' } ubuntu_nova_glance_cinder_services = { u'nova-api': u'nova-api', u'nova-cert': u'nova-cert', u'nova-compute': u'nova-compute', u'nova-conductor': u'nova-conductor', u'nova-consoleauth': u'nova-console', u'glance-api': u'glance-api', u'glance-reg': u'glance-reg', u'cinder-api': u'cinder-api', u'cinder-scheduler': u'cinder-scheduler' } result = dict() result = service_map.merge_data(self.nova_api_services, result, "ubuntu") self.assertDictEqual(result, ubuntu_nova_api_services, "Merge failed") result = service_map.merge_data(self.glance_api_services, result, "ubuntu") self.assertDictEqual(result, ubuntu_nova_glance_services, "Merge failed") result = service_map.merge_data(self.cinder_api_services, result, "ubuntu") self.assertDictEqual(result, ubuntu_nova_glance_cinder_services, "Merge failed") diskimage-builder-2.11.0/diskimage_builder/elements/sysctl/000077500000000000000000000000001324273057700237665ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/sysctl/README.rst000066400000000000000000000004101324273057700254500ustar00rootroot00000000000000====== sysctl ====== Add a sysctl-set-value command which can be run from within an element. Running this command will cause the sysctl value to be set on boot (by writing the value to /etc/sysctl.d). Example usage :: sysctl-set-value net.ipv4.ip_forward 1 diskimage-builder-2.11.0/diskimage_builder/elements/sysctl/bin/000077500000000000000000000000001324273057700245365ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/sysctl/bin/sysctl-set-value000077500000000000000000000022701324273057700277110ustar00rootroot00000000000000#!/bin/bash # # Copied from tripleo-image-element's sysctl element # # Validate and manage setting sysctl settings. # # The script is called with name/value pairs which are stored # in the system default sysctl.d directory. Before adding new # settings a validation is done to ensure that conflicting # sysctl settings have not been requested. Once finished sysctl # is used to activate the changes. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail NAME=${1:-} VALUE=${2:-} # Optional comment used to describe the setting COMMENT=${3:-"This file was created by diskimage-builder."} if [ -z "$NAME" -o -z "$VALUE" ]; then echo "NAME and VALUE are required." exit 1 fi FILENAME="/etc/sysctl.d/${NAME}.conf" if [ -f $FILENAME ]; then # check to make sure the settings match... otherwise fail if ! grep -q "^$NAME = $VALUE" $FILENAME; then echo "Conflicting sysctl.conf setting for $NAME == $VALUE. Found:" grep "^$NAME" $FILENAME exit 1 fi else if ! sysctl -a | grep -q "^$NAME"; then echo "Invalid sysctl key: $NAME" exit 1 fi sysctl-write-value $NAME "$VALUE" "$COMMENT" sysctl -p $FILENAME fi diskimage-builder-2.11.0/diskimage_builder/elements/sysctl/bin/sysctl-write-value000077500000000000000000000017431324273057700302540ustar00rootroot00000000000000#!/bin/bash # # Copied from tripleo-image-element's sysctl element # # Validate and manage setting sysctl settings. # # The script is called with name/value pairs which are stored # in the system default sysctl.d directory. This script performs # no checking, just writing out the file. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail NAME=${1:-} VALUE=${2:-} # Optional comment used to describe the setting COMMENT=${3:-"This file was created by diskimage-builder."} if [ -z "$NAME" -o -z "$VALUE" ]; then echo "Usage: sysctl-write-value [comment]" exit 1 fi FILENAME="/etc/sysctl.d/${NAME}.conf" if [ -f $FILENAME ]; then # check to make sure the settings match... otherwise fail if ! grep -q "^$NAME = $VALUE" $FILENAME; then echo "Conflicting sysctl.conf setting for $NAME == $VALUE. Found:" grep "^$NAME" $FILENAME exit 1 fi else cat > $FILENAME < /etc/machine-id fi diskimage-builder-2.11.0/diskimage_builder/elements/uboot/000077500000000000000000000000001324273057700235755ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/uboot/README.rst000066400000000000000000000006661324273057700252740ustar00rootroot00000000000000===== uboot ===== Perform kernel/initrd post-processing for UBoot. This element helps post-process the kernel/initrd for use with uboot. It works with ramdisk images as well as user images. This element needs u-boot-tools to be installed on the host. The load address and entry point for UBoot kernel can be specified as shown in the example below. Example: export UBOOT\_KERNEL\_ADDR=0x80000 export UBOOT\_KERNEL\_EP=0x80000 diskimage-builder-2.11.0/diskimage_builder/elements/uboot/cleanup.d/000077500000000000000000000000001324273057700254465ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/uboot/cleanup.d/98-uboot000077500000000000000000000016241324273057700267650ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail function post_process() { mkimage -A $1 -O linux -T kernel -C none -a $UBOOT_KERNEL_ADDR -e $UBOOT_KERNEL_EP -n "Kernel" -d $KERNEL uImage mv uImage $KERNEL mkimage -A $1 -O linux -T ramdisk -a 0 -n "RAMdisk" -C gzip -d $RAMDISK uInitrd mv uInitrd $RAMDISK } UBOOT_KERNEL_ADDR=${UBOOT_KERNEL_ADDR:-0x80000} UBOOT_KERNEL_EP=${UBOOT_KERNEL_EP:-0x80000} [ -n "$TARGET_ROOT" ] # Check if RAMDISK is being built. if [ -f "$TARGET_ROOT/tmp/ramdisk" ] ; then KERNEL="$TARGET_ROOT/tmp/kernel" RAMDISK="$TARGET_ROOT/tmp/ramdisk" else source "$_LIB/img-functions" # Dig up the initrd and kernel to use. select_boot_kernel_initrd "$TARGET_ROOT" KERNEL="$TARGET_ROOT/boot/$KERNEL" RAMDISK="$TARGET_ROOT/boot/$RAMDISK" fi case "$ARCH" in arm*) post_process arm ;; esac diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-common/000077500000000000000000000000001324273057700252555ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-common/README.rst000066400000000000000000000001611324273057700267420ustar00rootroot00000000000000ubuntu-common ============= This element holds configuration and scripts that are common for all Ubuntu images. diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-common/element-deps000066400000000000000000000000101324273057700275510ustar00rootroot00000000000000sysprep diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-common/install.d/000077500000000000000000000000001324273057700271455ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-common/install.d/80-disable-rfc3041000077500000000000000000000025711324273057700320100ustar00rootroot00000000000000#!/bin/bash # Copyright 2016, Rackspace US, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # # See the License for the specific language governing permissions and # limitations under the License. # dib-lint: disable=set setu setpipefail indent if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -e # This will disable the privacy extensions for IPv6 (RFC4941) # (RFC4941 obsoletes RFC3041) # # Note that depending on the implementation of how the sysctl # settings are applied there is a race condition: there might be # a small time-frame where the original Ubuntu settings from # /etc/sysctl.d/10-ipv6-privacy.conf are already applied but the # new settings from /etc/sysctl.d/99-cloudimg-ipv6.conf # not yet. # If during this time all started network interfaces will # use the privacy extension. cat > /etc/sysctl.d/99-cloudimg-ipv6.conf < $CACHED_FILE_LOCK diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/000077500000000000000000000000001324273057700254135ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/README.rst000066400000000000000000000011021324273057700270740ustar00rootroot00000000000000============== ubuntu-minimal ============== The ``ubuntu-minimal`` element uses debootstrap for generating a minimal image. In contrast the ``ubuntu`` element uses the cloud-image as the initial base. By default this element creates the latest LTS release. The exact setting can be found in the element's ``environment.d`` directory in the variable ``DIB_RELEASE``. If a different release of Ubuntu should be created, the variable ``DIB_RELEASE`` can be set appropriately. Note that this element installs ``systemd-sysv`` as the init system for 18.04+. .. element_deps:: diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/element-deps000066400000000000000000000000531324273057700277160ustar00rootroot00000000000000debootstrap package-installs ubuntu-common diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/element-provides000066400000000000000000000000211324273057700306110ustar00rootroot00000000000000operating-system diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/environment.d/000077500000000000000000000000001324273057700302015ustar00rootroot0000000000000010-ubuntu-distro-name.bash000066400000000000000000000007551324273057700347500ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/environment.dexport DISTRO_NAME=ubuntu export DIB_RELEASE=${DIB_RELEASE:-xenial} export DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main,universe} # There are two default distro mirrors depending on architecture ARCH=${ARCH:-} if [[ "arm64 armhf powerpc ppc64el s390x" =~ "$ARCH" ]]; then default_ubuntu_mirror=http://ports.ubuntu.com/ubuntu-ports else default_ubuntu_mirror=http://archive.ubuntu.com/ubuntu fi export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-$default_ubuntu_mirror} diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/package-installs.yaml000066400000000000000000000000461324273057700315210ustar00rootroot00000000000000linux-image-generic: initramfs-tools: diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/pre-install.d/000077500000000000000000000000001324273057700300675ustar00rootroot0000000000000001-ubuntu-apt-update000077500000000000000000000022701324273057700335410ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/pre-install.d#!/bin/bash # Copyright (c) 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # # See the License for the specific language governing permissions and # limitations under the License. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # We should manage this in a betterer way cat << EOF >/etc/apt/sources.list deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE ${DIB_DEBIAN_COMPONENTS//,/ } deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-updates ${DIB_DEBIAN_COMPONENTS//,/ } deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-backports ${DIB_DEBIAN_COMPONENTS//,/ } deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-security ${DIB_DEBIAN_COMPONENTS//,/ } EOF # Need to update to retrieve the signed Release file apt-get update apt-get clean apt-get dist-upgrade -y diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/root.d/000077500000000000000000000000001324273057700266205ustar00rootroot0000000000000075-ubuntu-minimal-baseinstall000077500000000000000000000041031324273057700341630ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/root.d#!/bin/bash # Copyright (c) 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # # See the License for the specific language governing permissions and # limitations under the License. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # We should manage this in a betterer way sudo bash -c "cat << EOF >$TARGET_ROOT/etc/apt/sources.list deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE ${DIB_DEBIAN_COMPONENTS//,/ } deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-updates ${DIB_DEBIAN_COMPONENTS//,/ } deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-backports ${DIB_DEBIAN_COMPONENTS//,/ } deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-security ${DIB_DEBIAN_COMPONENTS//,/ } EOF" sudo mount -t proc none $TARGET_ROOT/proc sudo mount -t sysfs none $TARGET_ROOT/sys trap "sudo umount $TARGET_ROOT/proc; sudo umount $TARGET_ROOT/sys" EXIT apt_get="sudo chroot $TARGET_ROOT /usr/bin/apt-get" # dib-lint: safe_sudo # Need to update to retrieve the signed Release file $apt_get update $apt_get clean $apt_get dist-upgrade -y to_install="" # pre-bionic (18.04) brought this in via debootstrap, but init # dependencies have narrowed in the container world, so now we add it # explicitly here so it's brought in early. if [ $DIB_RELEASE != "trusty" ] && [ $DIB_RELEASE != "xenial" ]; then to_install+="systemd-sysv " fi # default required to_install+="busybox sudo " # dib-lint: safe_sudo if [ "$DIB_PYTHON_VERSION" == "2" ]; then to_install+="python " elif [ "$DIB_PYTHON_VERSION" == "3" ]; then to_install+="python3 " else echo "ERROR: DIB_PYTHON_VERSION is '$DIB_PYTHON_VERSION' but needs to be 2 or 3" exit 1 fi $apt_get install -y $to_install diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/test-elements/000077500000000000000000000000001324273057700302045ustar00rootroot00000000000000trusty-build-succeeds/000077500000000000000000000000001324273057700343705ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/test-elementsREADME.rst000066400000000000000000000000541324273057700360560ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/test-elements/trusty-build-succeedsVerify we can build a ubuntu-minimal image. element-deps000066400000000000000000000000251324273057700366720ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/test-elements/trusty-build-succeedsopenstack-ci-mirrors environment.d/000077500000000000000000000000001324273057700371565ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/test-elements/trusty-build-succeeds10-set-distro.bash000066400000000000000000000000321324273057700423230ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/test-elements/trusty-build-succeeds/environment.dexport DIB_RELEASE=trusty xenial-build-succeeds/000077500000000000000000000000001324273057700342765ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/test-elementsREADME.rst000066400000000000000000000002251324273057700357640ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/test-elements/xenial-build-succeedsVerify we can build a ubuntu-minimal image. Note this test includes the vm element to test the bootloader install, and specifies to output a .qcow2 element-deps000066400000000000000000000000301324273057700365740ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/test-elements/xenial-build-succeedsopenstack-ci-mirrors vm environment.d/000077500000000000000000000000001324273057700370645ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/test-elements/xenial-build-succeeds10-set-distro.bash000066400000000000000000000000321324273057700422310ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/test-elements/xenial-build-succeeds/environment.dexport DIB_RELEASE=xenial test-output-formats000066400000000000000000000000111324273057700401770ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-minimal/test-elements/xenial-build-succeedstar,qcow2diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-signed/000077500000000000000000000000001324273057700252365ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-signed/README.rst000066400000000000000000000003261324273057700267260ustar00rootroot00000000000000============= ubuntu-signed ============= The ``ubuntu-signed`` element installs ``linux-signed-image-generic`` that provides signed kernel that can be used for deploy in UEFI secure boot mode. .. element_deps:: diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-signed/element-deps000066400000000000000000000000071324273057700275400ustar00rootroot00000000000000ubuntu diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-signed/package-installs.yaml000066400000000000000000000000341324273057700313410ustar00rootroot00000000000000linux-signed-image-generic: diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-signed/post-install.d/000077500000000000000000000000001324273057700301115ustar00rootroot0000000000000090-get-signed-kernel000077500000000000000000000021621324273057700335130ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu-signed/post-install.d#!/bin/bash # # Copyright 2015 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail # Get signed kernel version LATEST_SIGNED_KERNEL=$(ls /boot/vmlinu*.efi.signed | sort | tail -1) if [ "$LATEST_SIGNED_KERNEL" == "" ]; then echo "Unable to find a suitable kernel" >>/dev/stderr exit 1 fi SIGNED_KERNEL_VERSION=${LATEST_SIGNED_KERNEL##/boot/vmlinu[zx]-} echo "Found signed ubuntu kernel version $SIGNED_KERNEL_VERSION" cat > /dib-signed-kernel-version << EOF DIB_SIGNED_KERNEL_VERSION=$SIGNED_KERNEL_VERSION EOF diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/000077500000000000000000000000001324273057700237675ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/README.rst000066400000000000000000000006301324273057700254550ustar00rootroot00000000000000====== ubuntu ====== Use Ubuntu cloud images as the baseline for built disk images. Overrides: * To use a non-default URL for downloading base Ubuntu cloud images, use the environment variable ``DIB_CLOUD_IMAGES`` * To download a non-default release of Ubuntu cloud images, use the environment variable ``DIB_RELEASE``. This element will export the ``DIB_RELEASE`` variable. .. element_deps:: diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/element-deps000066400000000000000000000000711324273057700262720ustar00rootroot00000000000000cache-url cloud-init-datasources dkms dpkg ubuntu-common diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/element-provides000066400000000000000000000000211324273057700271650ustar00rootroot00000000000000operating-system diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/environment.d/000077500000000000000000000000001324273057700265555ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/environment.d/10-ubuntu-distro-name.bash000066400000000000000000000001041324273057700333670ustar00rootroot00000000000000export DISTRO_NAME=ubuntu export DIB_RELEASE=${DIB_RELEASE:-xenial} 99-cloud-init-datasources.bash000066400000000000000000000003621324273057700341570ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/environment.d# NOTE(adam_g): Until (LP: #1316475) is resolved in Ubuntu, default to only # allowing the Ec2 data source from being queried on first boot, unless # specified otherwise. export DIB_CLOUD_INIT_DATASOURCES=${DIB_CLOUD_INIT_DATASOURCES:-"Ec2"} diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/install.d/000077500000000000000000000000001324273057700256575ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/install.d/99-autoremove000077500000000000000000000001631324273057700302320ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail apt-get -y autoremove diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/package-installs.yaml000066400000000000000000000000251324273057700300720ustar00rootroot00000000000000linux-image-generic: diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/pre-install.d/000077500000000000000000000000001324273057700264435ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/pre-install.d/00-remove-apt-xapian-index000077500000000000000000000004151324273057700332500ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # This package is broken and causes real issues on update: # https://bugs.launchpad.net/ubuntu/+source/apt-xapian-index/+bug/1227420 apt-get --yes remove apt-xapian-index || : diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/pre-install.d/00-remove-grub000077500000000000000000000006111324273057700310360ustar00rootroot00000000000000#!/bin/bash # The grub post-kernel install hook will barf if the block device can't be # found (as happens in a chroot). # Temporarily remove grub, to avoid that confusion. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail if dpkg-query -W grub-pc; then apt-get -y remove grub-pc fi if dpkg-query -W grub2-common; then apt-get -y remove grub2-common fi diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/pre-install.d/01-set-ubuntu-mirror000077500000000000000000000004661324273057700322400ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-} [ -n "$DIB_DISTRIBUTION_MIRROR" ] || exit 0 sudo sed -ie "s&http://\(archive\|security\).ubuntu.com/ubuntu&$DIB_DISTRIBUTION_MIRROR&" \ /etc/apt/sources.list diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/root.d/000077500000000000000000000000001324273057700251745ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/root.d/10-cache-ubuntu-tarball000077500000000000000000000047311324273057700313470ustar00rootroot00000000000000#!/bin/bash # These are useful, or at worst not harmful, for all images we build. if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail [ -n "$ARCH" ] [ -n "$TARGET_ROOT" ] shopt -s extglob DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-http://cloud-images.ubuntu.com} DIB_RELEASE=${DIB_RELEASE:-trusty} BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-$DIB_RELEASE-server-cloudimg-$ARCH-root.tar.gz} SHA256SUMS=${SHA256SUMS:-https://${DIB_CLOUD_IMAGES##http?(s)://}/$DIB_RELEASE/current/SHA256SUMS} CACHED_FILE=$DIB_IMAGE_CACHE/$BASE_IMAGE_FILE CACHED_FILE_LOCK=$DIB_LOCKFILES/$BASE_IMAGE_FILE.lock CACHED_SUMS=$DIB_IMAGE_CACHE/SHA256SUMS.ubuntu.$DIB_RELEASE.$ARCH function get_ubuntu_tarball() { if [ -n "$DIB_OFFLINE" -a -f "$CACHED_FILE" ] ; then echo "Not checking freshness of cached $CACHED_FILE." else echo "Fetching Base Image" $TMP_HOOKS_PATH/bin/cache-url $SHA256SUMS $CACHED_SUMS $TMP_HOOKS_PATH/bin/cache-url \ $DIB_CLOUD_IMAGES/$DIB_RELEASE/current/$BASE_IMAGE_FILE $CACHED_FILE pushd $DIB_IMAGE_CACHE if ! grep "$BASE_IMAGE_FILE" $CACHED_SUMS | sha256sum --check - ; then # It is likely that an upstream http(s) proxy has given us a skewed # result - either a cached SHA file or a cached image. Use cache-busting # to get (as long as caches are compliant...) fresh files. # Try the sha256sum first, just in case that is the stale one (avoiding # downloading the larger image), and then if the sums still fail retry # the image. $TMP_HOOKS_PATH/bin/cache-url -f $SHA256SUMS $CACHED_SUMS if ! grep "$BASE_IMAGE_FILE" $CACHED_SUMS | sha256sum --check - ; then $TMP_HOOKS_PATH/bin/cache-url -f \ $DIB_CLOUD_IMAGES/$DIB_RELEASE/current/$BASE_IMAGE_FILE $CACHED_FILE grep "$BASE_IMAGE_FILE" $CACHED_SUMS | sha256sum --check - fi fi popd fi # Extract the base image (use --numeric-owner to avoid UID/GID mismatch between # image tarball and host OS e.g. when building Ubuntu image on an openSUSE host) sudo tar -C $TARGET_ROOT --numeric-owner -xzf $DIB_IMAGE_CACHE/$BASE_IMAGE_FILE } ( echo "Getting $CACHED_FILE_LOCK: $(date)" # Wait up to 20 minutes for another process to download if ! flock -w 1200 9 ; then echo "Did not get $CACHED_FILE_LOCK: $(date)" exit 1 fi get_ubuntu_tarball ) 9> $CACHED_FILE_LOCK diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/test-elements/000077500000000000000000000000001324273057700265605ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/test-elements/trusty-build-succeeds/000077500000000000000000000000001324273057700330235ustar00rootroot00000000000000README.rst000066400000000000000000000000441324273057700344310ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/test-elements/trusty-build-succeedsVerify we can build a ubuntu image. element-deps000066400000000000000000000000251324273057700352460ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/test-elements/trusty-build-succeedsopenstack-ci-mirrors environment.d/000077500000000000000000000000001324273057700355325ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/test-elements/trusty-build-succeeds10-set-distro.bash000066400000000000000000000000321324273057700406770ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/test-elements/trusty-build-succeeds/environment.dexport DIB_RELEASE=trusty diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/test-elements/xenial-build-succeeds/000077500000000000000000000000001324273057700327315ustar00rootroot00000000000000README.rst000066400000000000000000000002151324273057700343370ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/test-elements/xenial-build-succeedsVerify we can build a ubuntu image. Note this test includes the vm element to test the bootloader install, and specifies to output a .qcow2 element-deps000066400000000000000000000000301324273057700351500ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/test-elements/xenial-build-succeedsopenstack-ci-mirrors vm environment.d/000077500000000000000000000000001324273057700354405ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/test-elements/xenial-build-succeeds10-set-distro.bash000066400000000000000000000000321324273057700406050ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/test-elements/xenial-build-succeeds/environment.dexport DIB_RELEASE=xenial test-output-formats000066400000000000000000000000111324273057700365530ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/ubuntu/test-elements/xenial-build-succeedstar,qcow2diskimage-builder-2.11.0/diskimage_builder/elements/vm/000077500000000000000000000000001324273057700230675ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/vm/README.rst000066400000000000000000000001501324273057700245520ustar00rootroot00000000000000== vm == Sets up a partitioned disk (rather than building just one filesystem with no partition table). diskimage-builder-2.11.0/diskimage_builder/elements/vm/block-device-default.yaml000066400000000000000000000005301324273057700277220ustar00rootroot00000000000000# Default single partition loopback - local_loop: name: image0 - partitioning: base: image0 label: mbr partitions: - name: root flags: [ boot, primary ] size: 100% mkfs: mount: mount_point: / fstab: options: "defaults" fsck-passno: 1 diskimage-builder-2.11.0/diskimage_builder/elements/vm/block-device-ppc64el.yaml000066400000000000000000000017021324273057700275550ustar00rootroot00000000000000# Default block device setup for PPC # ---------------------------------- # # Details (extracted from [1]) # # Power Firmware (OFW) scans chosen (nvram value boot-device) device, # for either PReP* partition or vfat. # # PReP is raw and small (8M max) partition which caries only stage1 # binary. # msdos partition table: # - PReP partition have 0x41 type (must be active) # - PReP partition must have boot flag set # # [1] https://www.ibm.com/developerworks/community/wikis/home?lang=en#!/wiki/W51a7ffcf4dfd_4b40_9d82_446ebc23c550/page/PowerLinux%20Boot%20howto - local_loop: name: image0 - partitioning: base: image0 label: mbr partitions: - name: boot flags: [ boot, primary ] type: '0x41' size: 8MiB - name: root flags: [ primary ] size: 100% mkfs: mount: mount_point: / fstab: options: "defaults" fsck-passno: 1 diskimage-builder-2.11.0/diskimage_builder/elements/vm/element-deps000066400000000000000000000000131324273057700253660ustar00rootroot00000000000000bootloader diskimage-builder-2.11.0/diskimage_builder/elements/vm/finalise.d/000077500000000000000000000000001324273057700251035ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/vm/finalise.d/50-remove-bogus-udev-links000077500000000000000000000003621324273057700317450ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail if [ $DISTRO_NAME = 'opensuse' ] ; then # workaround for https://bugzilla.novell.com/show_bug.cgi?id=859493 rm -f /dev/mapper/loop*_part1 fi diskimage-builder-2.11.0/diskimage_builder/elements/yum-minimal/000077500000000000000000000000001324273057700247035ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/yum-minimal/README.rst000066400000000000000000000014631324273057700263760ustar00rootroot00000000000000=========== yum-minimal =========== Base element for creating minimal yum-based images. This element is incomplete by itself, you'll want to use the centos-minimal or fedora-minimal elements to get an actual base image. Use of this element will require 'yum' and 'yum-utils' to be installed on Ubuntu and Debian. Nothing additional is needed on Fedora or CentOS. If you wish to have DHCP networking setup for eth0 & eth1 via /etc/sysconfig/network-config scripts/ifcfg-eth[0|1], set the environment variable `DIB_YUM_MINIMAL_CREATE_INTERFACES` to `1`. If you wish to build from specific mirrors, set ``DIB_YUM_MINIMAL_BOOTSTRAP_REPOS`` to a directory with the ``.repo`` files to use during bootstrap and build. The repo files should be named with a prefix ``dib-mirror-`` and will be removed from the final image. diskimage-builder-2.11.0/diskimage_builder/elements/yum-minimal/cleanup.d/000077500000000000000000000000001324273057700265545ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/yum-minimal/cleanup.d/95-remove-yum-mirror000077500000000000000000000020341324273057700323510ustar00rootroot00000000000000#!/bin/bash # # Copyright 2015 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # # dib-lint: disable=safe_sudo if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail # This removes any build-time repos that might be set in 08-yum-chroot if [[ -n ${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS:-} ]]; then sudo rm -f $TMP_MOUNT_PATH/etc/yum.repos.d/dib-mirror*.repo for repo in $TMP_MOUNT_PATH/etc/yum.repos.d/*.USING_MIRROR; do sudo mv $repo ${repo/.USING_MIRROR/} done fi diskimage-builder-2.11.0/diskimage_builder/elements/yum-minimal/element-deps000066400000000000000000000000351324273057700272060ustar00rootroot00000000000000redhat-common rpm-distro yum diskimage-builder-2.11.0/diskimage_builder/elements/yum-minimal/install.d/000077500000000000000000000000001324273057700265735ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/yum-minimal/install.d/10-base-networking000077500000000000000000000026471324273057700320470ustar00rootroot00000000000000#!/bin/bash # # Copyright 2015 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail # allow networking init scripts inside the container to work without extra steps cat << EOF | sudo tee /etc/sysconfig/network > /dev/null NETWORKING=yes NETWORKING_IPV6=yes NOZEROCONF=yes EOF # If you want eth0 and eth1 created as DHCP based interfaces, enable # this. You don't want this if systemd is going to call the # interfaces on the real system something else, or if you're using a # network-manager like cloud-init, glean or network-manager that will # handle the interfaces dynamically. if [[ "${DIB_YUM_MINIMAL_CREATE_INTERFACES:-0}" -eq "1" ]]; then for interface in eth0 eth1; do cat << EOF | tee /etc/sysconfig/network-scripts/ifcfg-$interface > /dev/null DEVICE=$interface BOOTPROTO=dhcp ONBOOT=on EOF done fi diskimage-builder-2.11.0/diskimage_builder/elements/yum-minimal/package-installs.yaml000066400000000000000000000001551324273057700310120ustar00rootroot00000000000000audit: dracut: dracut-config-generic: grubby: kernel: initscripts: man-pages: lsb_release: redhat-rpm-config:diskimage-builder-2.11.0/diskimage_builder/elements/yum-minimal/pkg-map000066400000000000000000000001241324273057700261570ustar00rootroot00000000000000{ "family": { "redhat": { "lsb_release": "redhat-lsb-core" } } } diskimage-builder-2.11.0/diskimage_builder/elements/yum-minimal/pre-install.d/000077500000000000000000000000001324273057700273575ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/yum-minimal/pre-install.d/03-yum-cleanup000077500000000000000000000060051324273057700317650ustar00rootroot00000000000000#!/bin/bash # # Copyright 2015 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail # effectively: febootstrap-minimize --keep-zoneinfo --keep-rpmdb --keep-services "$target" # This is only required on CentOS ... see notes in # root.d/08-yum-chroot about %_install_langs if [[ $DISTRO_NAME != "fedora" ]]; then # Stripping *all* locales is a bad idea. For now, we take the # suggestion in [1] for reducing this # [1] https://bugzilla.redhat.com/show_bug.cgi?id=156477 if [ ! -f /usr/lib/locale/locale-archive ]; then die "locale-archive not found? Can not do cleanup." fi # now the archive has everything in it, and is about 100MiB. Strip it # to just en_US (basically, this is the locale we support if you ssh # in, other than POSIX) localedef --delete-from-archive \ $(localedef --list-archive | grep -v '^en_US' | xargs) # This removes the locales from the archive index but doesn't rebuild # the file, so it is still the same size (maybe it is sparse? # presumably as it's mmapped you don't want to fiddle with the offsets # of locales in the archive on a live system. We are not live). # build-locale-archive is a tool provided by the RH packaging of # glibc. Documentation is scarce, but it takes the pre-built locales # in the tmpl file and creates an archive. It seems originally the # installer would set some flags to rpm to tell the package what # languages to pick out of the template, but along the way, this was # reverted to install them (size considered less important than locale # support). # We hack this by moving the locale-archive we've removed the extra # locales from to the template, then re-run the build. mv /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.tmpl # rebuild archive from template /usr/sbin/build-locale-archive # leave empty template behind as package does. I think this stops # upgrades redoing things echo > /usr/lib/locale/locale-archive.tmpl # remove the unnecessary source locale files and iconv files pushd /usr/share/locale find . ! -name 'locale.alias' -delete popd rm -rf {lib.lib64}/gconv fi # docs rm -rf /usr/share/{doc,info,gnome/help} # cracklib rm -rf /usr/share/cracklib # i18n rm -rf /usr/share/i18n # sln rm -rf /sbin/sln # ldconfig rm -rf /etc/ld.so.cache /var/cache/ldconfig mkdir -p --mode=0755 /var/cache/ldconfig diskimage-builder-2.11.0/diskimage_builder/elements/yum-minimal/root.d/000077500000000000000000000000001324273057700261105ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/yum-minimal/root.d/08-yum-chroot000077500000000000000000000316351324273057700304010ustar00rootroot00000000000000#!/bin/bash # # Copyright 2015 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # # dib-lint: disable=safe_sudo if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail source $_LIB/common-functions if [ -f ${TARGET_ROOT}/.extra_settings ] ; then . ${TARGET_ROOT}/.extra_settings fi ARCH=${ARCH:-x86_64} if [ $ARCH = amd64 ]; then ARCH=x86_64 fi # Calling elements will need to set DISTRO_NAME and DIB_RELEASE # TODO Maybe deal with DIB_DISTRIBUTION_MIRROR http_proxy=${http_proxy:-} YUM=${YUM:-yum} WORKING=$(mktemp --tmpdir=${TMP_DIR:-/tmp} -d) EACTION="rm -r $WORKING" trap "$EACTION" EXIT YUM_CACHE=$DIB_IMAGE_CACHE/yum mkdir -p $YUM_CACHE # Note, on Debian/Ubuntu, %_dbpath is set in the RPM macros as # ${HOME}/.rpmdb/ -- this makes sense as RPM isn't the system # packager. This path is relative to the "--root" argument _RPM="rpm --dbpath=/var/lib/rpm" # install the [fedora|centos]-[release|repo] packages inside the # chroot, which are needed to bootstrap yum/dnf # # note this runs outside the chroot, where we're assuming the platform # has yum/yumdownloader function _install_repos { local packages local rc # pre-install the base system packages via rpm. We previously # just left it up to yum to drag these in when we "yum install # yum" in the chroot in _install_pkg_manager. This raised a small # problem that inside the empty chroot yum went ahead and did a # mkdir for /var/run to put some pid file in, which then messed up # the "filesystem" package making /var/run a symlink to /run # ... which leads to odd issues with a running system. # # TODO: these packages still have some small %posttrans stuff that # depends on other packages (see rhbz#1306489) ... maybe the idea # is that they are only installed in one big transaction with the # rest of the system? but we don't want to use yum to do this # (see above) so ... packages="basesystem filesystem setup " packages+="${DISTRO_NAME}-release " # Starting in F21 this was split into a separate package if [[ ${DISTRO_NAME} == 'fedora' ]]; then packages+="fedora-repos " fi # F27 started putting gpg keys into this separate package if [[ ${DISTRO_NAME} = fedora && ${DIB_RELEASE} -gt 26 ]]; then packages+="fedora-gpg-keys " fi # By default, parent elements (fedora-minimal, centos-minimal) # have a yum.repos.d directory in the element with a default repo; # this is copied to TMP_HOOK_PATH by the usual hook-copying # routines. In the gate, environment.d files for the funtional # tests will set DIB_YUM_MINIMAL_BOOTSTRAP_REPOS -- this contains # mirrors correct for the region setup by contrib/setup-gate-mirrors.sh local repo=${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS:-${TMP_HOOKS_PATH}/yum.repos.d} # yumdownloader puts repo xml files and such into a directory # ${TMPDIR}/yum-$USER-random. Since we don't need this once the # initial download happens, redirect TMPDIR for this call so we # can clean it up nicely local temp_tmp temp_tmp=$(mktemp -d) TMPDIR=${temp_tmp} yumdownloader --verbose \ --releasever=$DIB_RELEASE \ --setopt=reposdir=$repo \ --destdir=$WORKING \ ${packages} && rc=$? || rc=$? rm -rf ${temp_tmp} if [[ ${rc} != 0 ]]; then die "Failed to download initial packages: ${packages}" fi # --nodeps works around these wanting /bin/sh in some fedora # releases, see rhbz#1265873 sudo $_RPM --root $TARGET_ROOT --nodeps -ivh $WORKING/*rpm # install the bootstrap mirror repos over the default ones, if # set. we will remove this at the end so the final image has # regular mirrors if [[ -n ${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS:-} ]]; then for repo in $TARGET_ROOT/etc/yum.repos.d/*.repo; do sudo mv $repo $repo.USING_MIRROR done sudo cp ${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS}/* \ $TARGET_ROOT/etc/yum.repos.d/ fi } # _install_pkg_manager packages... # # install the package manager packages. This is done outside the chroot # and with yum from the build system. # TODO: one day build systems will be dnf only, but we don't handle # that right now function _install_pkg_manager { # Install into the chroot, using the gpg keys from the release # rpm's installed in the chroot sudo sed -i "s,/etc/pki/rpm-gpg,$TARGET_ROOT/etc/pki/rpm-gpg,g" \ $TARGET_ROOT/etc/yum.repos.d/*repo # See notes on $_RPM variable -- we need to override the # $HOME-based dbpath set on debian/ubuntu here. Unfortunately, # yum does not have a way to override rpm macros from the command # line. So we modify the user's ~/.rpmmacros to set %_dbpath back # to "/var/lib/rpm" (note, this is taken relative to the # --installroot). # # Also note, we only want this done around this call -- this is # the only place we are using yum outside the chroot, and hence # picking up the base-system's default rpm macros. For example, # the yumdownloader calls above in _install_repos want to use # ~/.rpmdb/ ... there is nothing in the build-system /var/lib/rpm! # # Another issue we hit is having to set --releasedir here. yum # determines $releasevar based on (more or less) "rpm -q # --whatprovides $distroverpkg". By default, this is # "redhat-release" (fedora-release provides redhat-release) but # some platforms like CentOS override it in /etc/yum.conf (to # centos-release in their case). You can't override this (see # [1]), but setting --releasever works around this. # # [1] https://bugzilla.redhat.com/show_bug.cgi?id=1287333 ( flock -w 1200 9 || die "Can not lock .rpmmacros" echo "%_dbpath /var/lib/rpm" >> $HOME/.rpmmacros local _lang_pack="" local _extra_pkgs="" if [ $DISTRO_NAME = "fedora" -a $DIB_RELEASE -le 23 ]; then # _install_langs is a rpm macro that limits the translation # files, etc installed by packages. For Fedora 23 [1], the # glibc-common package will obey this to only install the # listed locales, keeping things much smaller (we still have # to clean up locales manually on centos7). We install just # en_US because people often ssh in with that locale, but # leave out everything else. Note that yum has an option to # set this from the command-line [2], but the yum in trusty we # are using is too old to have it. So we set it directly in # the macros file # # [1] http://pkgs.fedoraproject.org/cgit/rpms/glibc.git/commit/glibc.spec?h=f23&id=91764bd9ec690d4b8a886c0a3a104aac12d340d2 # [2] http://yum.baseurl.org/gitweb?p=yum.git;a=commit;h=26128173b362474456e8f0642073ecb0322ed031 echo "%_install_langs C:en_US:en_US.UTF-8" >> $HOME/.rpmmacros elif [ $DISTRO_NAME = "fedora" -a $DIB_RELEASE -ge 24 ]; then # glibc on F24 has split locales into "langpack" packages. # Yum doesn't understand the weak-dependencies glibc now # uses to get the minimal-langpack and chooses a # random(ish) one that satisfies the locale dependency # (rhbz#1349258). Work-around this by explicitly requring # the minimal and english (for en_US.UTF-8) pack. _lang_pack="glibc-minimal-langpack glibc-langpack-en" fi # Yum has some issues choosing weak dependencies. It can end # up choosing "coreutils-single" instead of "coreutils" which # causes problems later when a package actually requires # coreutils. For more info see # https://bugzilla.redhat.com/show_bug.cgi?id=1286445 # Really all we can do is pre-install the right thing _extra_pkgs+="coreutils " sudo -E yum -y \ --disableexcludes=all \ --setopt=cachedir=$YUM_CACHE/$ARCH/$DIB_RELEASE \ --setopt=reposdir=$TARGET_ROOT/etc/yum.repos.d \ --releasever=$DIB_RELEASE \ --installroot $TARGET_ROOT \ install $@ ${_lang_pack} ${_extra_pkgs} && rc=$? || rc=$? # Note we've modified the base system's .rpmmacros. Ensure we # clean it up *always* # sed makes it easy to remove last line, but not last n lines... sed -i '$ d' $HOME/.rpmmacros; sed -i '$ d' $HOME/.rpmmacros; if [ $rc != 0 ]; then die "Initial yum install to chroot failed! Can not continue." fi ) 9>$DIB_LOCKFILES/.rpmmacros.dib.lock # Set gpg path back because subsequent actions will take place in # the chroot sudo sed -i "s,$TARGET_ROOT/etc/pki/rpm-gpg,/etc/pki/rpm-gpg,g" \ $TARGET_ROOT/etc/yum.repos.d/*repo } # Note this is not usually done for root.d elements (see # lib/common-functions:mount_proc_dev_sys) but it's important that # we have things like /dev/urandom around inside the chroot for # the rpm [pre|post]inst scripts within the packages. sudo mkdir -p $TARGET_ROOT/proc $TARGET_ROOT/dev $TARGET_ROOT/sys sudo mount -t proc none $TARGET_ROOT/proc sudo mount --bind /dev $TARGET_ROOT/dev sudo mount -t devpts $(mount_dev_pts_options) devpts $TARGET_ROOT/dev/pts sudo mount -t sysfs none $TARGET_ROOT/sys # initalize rpmdb sudo mkdir -p $TARGET_ROOT/var/lib/rpm sudo $_RPM --root $TARGET_ROOT --initdb # this makes sure that running yum/dnf in the chroot it can get # out to download stuff sudo mkdir $TARGET_ROOT/etc sudo cp /etc/resolv.conf $TARGET_ROOT/etc/resolv.conf # Bind mount the external yum cache inside the chroot. Same logic # as in the yum element to provide for yum caching copied here # because the sequencing is wrong otherwise sudo mkdir -p $TMP_MOUNT_PATH/tmp/yum sudo mount --bind $YUM_CACHE $TMP_MOUNT_PATH/tmp/yum _install_repos # install dnf for >= f22 if [ $DIB_RELEASE -ge 22 ]; then # Fedora 27 has a "curl-minimal" package that will get pulled in as a # rpm dependency. This causes problems later if/when "curl" gets # installed. To avoid this, just install the full curl first up. _install_pkg_manager dnf dnf-plugins-core yum curl else _install_pkg_manager yum fi # we just installed yum/dnf with "outside" tools (yum/rpm) which # might have created /var/lib/[yum|rpm] (etc) that are slighlty # incompatible. Refresh everything with the in-chroot tools sudo -E chroot $TARGET_ROOT rpm --rebuilddb sudo -E chroot $TARGET_ROOT ${YUM} clean all # populate the lang reduction macro in the chroot echo "%_install_langs C:en_US:en_US.UTF-8" | \ sudo tee -a $TARGET_ROOT/etc/rpm/macros.langs > /dev/null _base_packages="systemd passwd findutils sudo util-linux-ng " # This package is split out from systemd on >F24, dracut is # missing the dependency and will fail to make an initrd without # it; see # https://bugzilla.redhat.com/show_bug.cgi?id=1398505 _base_packages+="systemd-udev " # bootstrap the environment within the chroot; bring in new # metadata with an update and install some base packages we need. sudo -E chroot $TARGET_ROOT ${YUM} -y update sudo -E chroot $TARGET_ROOT ${YUM} -y \ --setopt=cachedir=/tmp/yum/$ARCH/$DIB_RELEASE \ install ${_base_packages} # Put in a dummy /etc/resolv.conf over the temporary one we used # to bootstrap. systemd has a bug/feature [1] that it will assume # you want systemd-networkd as the network manager and create a # broken symlink to /run/... if the base image doesn't have one. # This broken link confuses things like dhclient. # [1] https://bugzilla.redhat.com/show_bug.cgi?id=1197204 echo -e "# This file intentionally left blank\n" | \ sudo tee $TARGET_ROOT/etc/resolv.conf # set the most reliable UTF-8 locale echo -e 'LANG="en_US.UTF-8"' | \ sudo tee $TARGET_ROOT/etc/locale.conf # default to UTC sudo -E chroot $TARGET_ROOT ln -sf /usr/share/zoneinfo/UTC \ /etc/localtime # cleanup # TODO : move this into a exit trap; and reconsider how # this integrates with the global exit cleanup path. sudo umount $TMP_MOUNT_PATH/tmp/yum sudo umount $TARGET_ROOT/proc sudo umount $TARGET_ROOT/dev/pts sudo umount $TARGET_ROOT/dev sudo umount $TARGET_ROOT/sys # RPM doesn't know whether files have been changed since install # At this point though, we know for certain that we have changed no # config files, so anything marked .rpmnew is just a bug. for newfile in $(sudo find $TARGET_ROOT -type f -name '*rpmnew') ; do sudo mv $newfile $(echo $newfile | sed 's/.rpmnew$//') done sudo rm -f ${TARGET_ROOT}/.extra_settings diskimage-builder-2.11.0/diskimage_builder/elements/yum/000077500000000000000000000000001324273057700232575ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/yum/README.rst000066400000000000000000000015161324273057700247510ustar00rootroot00000000000000=== yum === Provide yum specific image building glue. RHEL/Fedora/CentOS and other yum based distributions need specific yum customizations. Customizations include caching of downloaded yum packages outside of the build chroot so that they can be reused by subsequent image builds. The cache increases image building speed when building multiple images, especially on slow connections. This is more effective than using an HTTP proxy as a yum cache since the same rpm from different mirrors is often requested. Custom yum repository configurations can also be applied by defining `DIB_YUM_REPO_CONF` to a space separated list of repo configuration files. The files will be copied to /etc/yum.repos.d/ during the image build, and then removed at the end of the build. Each repo file should be named differently to avoid a filename collision. diskimage-builder-2.11.0/diskimage_builder/elements/yum/bin/000077500000000000000000000000001324273057700240275ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/yum/bin/install-packages000077500000000000000000000107071324273057700272040ustar00rootroot00000000000000#!/bin/bash # Copyright 2012 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [ ${DIB_DEBUG_TRACE:-0} -gt 1 ]; then set -x fi set -eu set -o pipefail EXTRA_ARGS= ACTION=install MAP_ELEMENT= # allow override for dnf, as shipped by default with >=F22 YUM=${YUM:-yum} # save global xtrace state _xtrace=$(set +o | grep xtrace) SCRIPTNAME=$(basename $0) function show_options () { echo "Usage: $SCRIPTNAME [package ...]" echo echo "Options:" echo " -u -- update all packages" echo " -d dir -- download the packages to directory" echo " -e -- erase/remove packages" echo " -m map -- use custom element package map (Example: -m nova)" exit 0 } while getopts "hud:em:" opt; do case "$opt" in u) ${YUM} -y update; exit 0 ;; d) ACTION="download" DOWNLOAD_PATH=$OPTARG ;; e) ACTION="erase" ;; m) MAP_ELEMENT=$OPTARG ;; h) show_options ;; *) exit 1 ;; esac done shift $((OPTIND-1)) # Packages that aren't available in the distro but requested for installation # can be ignored by adding them to the exclude list BLACKLIST=$(cat /tmp/yum-blacklist 2>/dev/null || echo "") WHITELIST="" for i in "$@" do PKG_NAME=$i if [ -n "$MAP_ELEMENT" ]; then if ! PKG_NAME=$(pkg-map --element $MAP_ELEMENT $i); then echo "bin/pkg-map error. $PKG_NAME" exit 1 fi fi if [[ ! ${BLACKLIST[*]} =~ $PKG_NAME ]]; then WHITELIST="$WHITELIST $i" else echo "The package $i is not available and will not be installed" fi done if [ -n "$WHITELIST" ]; then if [ -f /etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release ]; then rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release fi if [ -n "$MAP_ELEMENT" ]; then if ! PKGS=$(pkg-map --element $MAP_ELEMENT $WHITELIST); then echo "bin/pkg-map error. $PKGS" exit 1 fi else PKGS=$(map-packages $WHITELIST) fi if [ -z "${PKGS}" ]; then echo "Not running install-packages $ACTION with empty packages list" exit 0 fi echo "Running install-packages ${ACTION}." if [ "$ACTION" == "download" ]; then mkdir -p $DOWNLOAD_PATH if [ ${YUM} == "dnf" ]; then dnf download --destdir=$DOWNLOAD_PATH $PKGS else # note; you don't want to use yum --download only here. # Firstly that only puts things in the yum cache # directory, and secondly it acts funny if old versions # are already in the cache. if [ ! -f /usr/bin/yumdownloader ]; then yum install -y yum-utils fi yumdownloader --destdir=$DOWNLOAD_PATH $PKGS fi exit 0 fi # yum & dnf have a feature/bug where missing packages in a list of # packages for install/erase do not raise an error exit [1]. # There is a corner case when specifying just *one* package, where # it will fail if the package is missing. Thus install-packages # follows this behaviour; we may need to re-evaluate this at some # point. # # [1] https://bugzilla.redhat.com/show_bug.cgi?id=965567 set -o xtrace ${YUM} -v -y $ACTION $EXTRA_ARGS $PKGS if [ "$ACTION" == "install" ]; then if [ ${YUM} == "dnf" ]; then # Make sure dnf won't autoremove these packages dnf mark install $PKGS fi fi $_xtrace # probably not the right place for this; but python-pip package on # fedora/rh calls pip "pip-python" while the rest of the work # expects it to be just called "pip" for pkg in "$@"; do if [ "$pkg" = "python-pip" ] ; then alternatives --install /usr/bin/pip pip /usr/bin/pip-python 10 fi done fi diskimage-builder-2.11.0/diskimage_builder/elements/yum/cleanup.d/000077500000000000000000000000001324273057700251305ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/yum/cleanup.d/99-remove-yum-repo-conf000077500000000000000000000006201324273057700313060ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail # exit directly if DIB_YUM_REPO_CONF is not defined properly if [ -z "${DIB_YUM_REPO_CONF:-}" ] ; then echo "DIB_YUM_REPO_CONF is not set - no repo configurations will be cleaned up" exit 0 fi for file in $DIB_YUM_REPO_CONF; do sudo rm -f $TMP_MOUNT_PATH/etc/yum.repos.d/$(basename $file) done diskimage-builder-2.11.0/diskimage_builder/elements/yum/element-deps000066400000000000000000000000141324273057700255570ustar00rootroot00000000000000install-bin diskimage-builder-2.11.0/diskimage_builder/elements/yum/extra-data.d/000077500000000000000000000000001324273057700255335ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/yum/extra-data.d/99-yum-repo-conf000077500000000000000000000013271324273057700304230ustar00rootroot00000000000000#!/bin/bash # Add additional yum repo configuration(s) with $DIB_YUM_REPO_CONF if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail # exit directly if DIB_YUM_REPO_CONF is not defined properly if [ -z "${DIB_YUM_REPO_CONF:-}" ] ; then echo "DIB_YUM_REPO_CONF is not set - no repo configuration will be copied in" exit 0 fi for file in $DIB_YUM_REPO_CONF; do if [ ! -f $file ]; then echo "$file is not a valid yum repo configuration file." echo "You should assign a list of proper yum repo configuration" echo "files in DIB_YUM_REPO_CONF." exit 1 fi # copy the yum repo configuration sudo cp -L -f $file $TMP_MOUNT_PATH/etc/yum.repos.d done diskimage-builder-2.11.0/diskimage_builder/elements/yum/post-install.d/000077500000000000000000000000001324273057700261325ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/yum/post-install.d/99-reset-yum-conf000077500000000000000000000005451324273057700312000ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail if [[ $DISTRO_NAME == "fedora" && $DIB_RELEASE -ge 22 ]]; then cfg=/etc/dnf/dnf.conf cachedir=/var/cache/dnf else cfg=/etc/yum.conf cachedir=/var/cache/yum fi sed -i "/^keepcache/c\keepcache=0" $cfg sed -i "/^cachedir/c\cachedir=${cachedir}" $cfg diskimage-builder-2.11.0/diskimage_builder/elements/yum/pre-install.d/000077500000000000000000000000001324273057700257335ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/yum/pre-install.d/00-dnf-update000077500000000000000000000004051324273057700301240ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail if [[ $DISTRO_NAME == "fedora" && $DIB_RELEASE -eq 22 ]]; then # we need dnf mark command from dnf 1.1.1, which is pretty recent dnf --refresh update -y dnf fi diskimage-builder-2.11.0/diskimage_builder/elements/yum/pre-install.d/01-yum-keepcache000077500000000000000000000007111324273057700306160ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail if [[ $DISTRO_NAME == "fedora" && $DIB_RELEASE -ge 22 ]]; then # dnf is slightly different; doesn't have these by default cfg=/etc/dnf/dnf.conf echo "keepcache=1" >> $cfg echo "cachedir=/tmp/yum" >> $cfg else cfg=/etc/yum.conf sed -i 's/keepcache=0/keepcache=1/' $cfg sed -i 's/cachedir=\/var\/cache\/yum/cachedir=\/tmp\/yum/' $cfg fi diskimage-builder-2.11.0/diskimage_builder/elements/yum/root.d/000077500000000000000000000000001324273057700244645ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/yum/root.d/50-yum-cache000077500000000000000000000004171324273057700265110ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail [ -n "$TARGET_ROOT" ] YUM_CACHE_DIR=$DIB_IMAGE_CACHE/yum mkdir -p $YUM_CACHE_DIR sudo mkdir -p $TMP_MOUNT_PATH/tmp/yum sudo mount --bind $YUM_CACHE_DIR $TMP_MOUNT_PATH/tmp/yum diskimage-builder-2.11.0/diskimage_builder/elements/zipl/000077500000000000000000000000001324273057700234235ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/zipl/README.rst000066400000000000000000000010471324273057700251140ustar00rootroot00000000000000========== zipl ========== Zipl is the bootloader for s390x. This element installs zipl on the base device holding the /boot directory on the system. It's mandatory for building s390x images. It replaces the `bootloader` element (which would install grub2 by default). This element has been tested with `ubuntu` and `ubuntu-minimal` distro. Arguments ========= * ``DIB_ZIPL_DEFAULT_CMDLINE`` sets the CMDLINE parameters that are appended to the zipl.conf parameter configuration. It defaults to 'LANG=en_US.UTF-8 console=ttyS0 console=ttyS1' diskimage-builder-2.11.0/diskimage_builder/elements/zipl/element-deps000066400000000000000000000000211324273057700257210ustar00rootroot00000000000000package-installs diskimage-builder-2.11.0/diskimage_builder/elements/zipl/element-provides000066400000000000000000000000131324273057700266220ustar00rootroot00000000000000bootloader diskimage-builder-2.11.0/diskimage_builder/elements/zipl/environment.d/000077500000000000000000000000001324273057700262115ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/zipl/environment.d/10-zipl-default-cmdline000066400000000000000000000001541324273057700323630ustar00rootroot00000000000000export DIB_ZIPL_DEFAULT_CMDLINE=${DIB_ZIPL_DEFAULT_CMDLINE:-"LANG=en_US.UTF-8 console=ttyS0 console=ttyS1"} diskimage-builder-2.11.0/diskimage_builder/elements/zipl/finalise.d/000077500000000000000000000000001324273057700254375ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/zipl/finalise.d/50-zipl000077500000000000000000000037251324273057700265740ustar00rootroot00000000000000#!/bin/bash # Copyright (c) 2017 IBM Corp. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # # See the License for the specific language governing permissions and # limitations under the License. # Configure zipl. if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then set -x fi set -eu set -o pipefail # get the device containing the /boot directory base_device=`df /boot --output=source | grep "/dev/*"` cat > /lib/s390-tools/zipl_helper.loop < /etc/zipl.conf <>"$KERNEL_CONF_FILE" diskimage-builder-2.11.0/diskimage_builder/elements/zypper-minimal/000077500000000000000000000000001324273057700254225ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/zypper-minimal/README.rst000066400000000000000000000003751324273057700271160ustar00rootroot00000000000000============== zypper-minimal ============== Base element for creating minimal SUSE-based images This element is incomplete by itself so you probably want to use it along with the opensuse-minimal one. It requires 'zypper' to be installed on the host. diskimage-builder-2.11.0/diskimage_builder/elements/zypper-minimal/element-deps000066400000000000000000000000301324273057700277200ustar00rootroot00000000000000package-installs zypper diskimage-builder-2.11.0/diskimage_builder/elements/zypper-minimal/package-installs.yaml000066400000000000000000000004001324273057700315220ustar00rootroot00000000000000# kernel kernel-default: # locale support (we need LC_ALL=en_US.utf8) glibc-locale: # And a few useful tools. Some are pulled # as dependencies but that may change so lets # be explicit. bash: ca-certificates-mozilla: lsb-release: openssl: sed: sudo: tar: diskimage-builder-2.11.0/diskimage_builder/elements/zypper-minimal/post-install.d/000077500000000000000000000000001324273057700302755ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/zypper-minimal/post-install.d/01-locale-cleanup000077500000000000000000000004351324273057700333270ustar00rootroot00000000000000#!/bin/bash # Only keep the minimum amount of locales to save disk space # if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail find /usr/lib/locale -mindepth 1 -maxdepth 1 -type d 2>/dev/null | \ grep -Ev '/(C.utf8|en_US|en_US.utf8)$' | xargs rm -rf diskimage-builder-2.11.0/diskimage_builder/elements/zypper-minimal/pre-install.d/000077500000000000000000000000001324273057700300765ustar00rootroot0000000000000000-disable-zypper-recommends000077500000000000000000000006401324273057700352460ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/zypper-minimal/pre-install.d#!/bin/bash # Prevent libsolv/zypper from installing recommended packages if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail fgrep -qx "solver.onlyRequires = true" /etc/zypp/zypp.conf || { cat >> /etc/zypp/zypp.conf << _EOF_ ## Disable installation of recommended packages, language packages ## and packages which depend on hardware (modalias). solver.onlyRequires = true _EOF_ } diskimage-builder-2.11.0/diskimage_builder/elements/zypper-minimal/root.d/000077500000000000000000000000001324273057700266275ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/zypper-minimal/root.d/08-zypper-chroot000077500000000000000000000117441324273057700316360ustar00rootroot00000000000000#!/bin/bash # # Copyright 2016 SUSE Linux GmbH # Copyright 2015 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # # dib-lint: disable=safe_sudo if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then set -x fi set -eu set -o pipefail source $_LIB/common-functions function cleanup() { sudo umount $TARGET_ROOT/proc sudo umount $TARGET_ROOT/dev/pts sudo umount $TARGET_ROOT/dev sudo umount $TARGET_ROOT/sys sudo umount $TMP_MOUNT_PATH/var/cache/zypp } trap cleanup EXIT ZYPPER_TARGET_OPTS="--non-interactive --gpg-auto-import-keys --root $TARGET_ROOT" ZYPPER_INSTALL_OPTS="--no-recommends" DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://download.opensuse.org} case ${DIB_RELEASE} in # We are using "=>" as the assignment symbol since "@" "=" etc could be used in the URI itself. # Remember, we can't export an array in bash so we use a string instead. # Repo format: {name}=>{uri} # Old openSUSE releases 13*) ZYPPER_REPOS="update=>${DIB_DISTRIBUTION_MIRROR}/update/${DIB_RELEASE}/ " ZYPPER_REPOS+="oss=>${DIB_DISTRIBUTION_MIRROR}/distribution/${DIB_RELEASE}/repo/oss/" ;; # New Leap releases 42*) ZYPPER_REPOS="update=>${DIB_DISTRIBUTION_MIRROR}/update/leap/${DIB_RELEASE}/oss/ " ZYPPER_REPOS+="oss=>${DIB_DISTRIBUTION_MIRROR}/distribution/leap/${DIB_RELEASE}/repo/oss/" ;; # Tumbleweed tumbleweed) ZYPPER_REPOS="update=>${DIB_DISTRIBUTION_MIRROR}/update/${DIB_RELEASE}/ " ZYPPER_REPOS+="oss=>${DIB_DISTRIBUTION_MIRROR}/${DIB_RELEASE}/repo/oss/" ;; *) echo "Unsupported openSUSE release: ${DIB_RELEASE}"; exit 1 ;; esac for repo in ${ZYPPER_REPOS}; do refresh_repo="" reponame=repo-${repo%%=>*} repouri=${repo##*=>} # Refresh all repos in TW and only the update one for the stable ones. This will ensure that # we always get the latest information from the repo. [[ ${DIB_RELEASE} == tumbleweed ]] || [[ ${reponame} == "repo-update" ]] && refresh_repo="-f" sudo zypper ${ZYPPER_TARGET_OPTS} addrepo --name ${reponame} --keep-packages ${refresh_repo} ${repouri} ${reponame} done # It appears that zypper will clean up the repo's cache when it (re-)adds the # repo so we need to add the cache now, once the repos are added. This is # similar to what the zypper/50-zypper-cache script does ZYPPER_CACHE_DIR=$DIB_IMAGE_CACHE/zypper mkdir -p $ZYPPER_CACHE_DIR sudo mkdir -p $TMP_MOUNT_PATH/var/cache/zypp sudo mount --bind $ZYPPER_CACHE_DIR $TMP_MOUNT_PATH/var/cache/zypp # Refresh it so we get updated data in cased we switched DIB_RELEASE # since last run. sudo zypper ${ZYPPER_TARGET_OPTS} refresh # Note this is not usually done for root.d elements (see # lib/common-functions:mount_proc_dev_sys) but it's important that # we have things like /dev/urandom around inside the chroot for # the rpm [pre|post]inst scripts within the packages. sudo mkdir -p $TARGET_ROOT/proc $TARGET_ROOT/dev $TARGET_ROOT/sys sudo mount -t proc none $TARGET_ROOT/proc sudo mount --bind /dev $TARGET_ROOT/dev sudo mount -t devpts $(mount_dev_pts_options) devpts $TARGET_ROOT/dev/pts sudo mount -t sysfs none $TARGET_ROOT/sys # Install filesystem, base and useful tools sudo zypper ${ZYPPER_TARGET_OPTS} install ${ZYPPER_INSTALL_OPTS} filesystem # Install basic components in order sudo zypper ${ZYPPER_TARGET_OPTS} install ${ZYPPER_INSTALL_OPTS} ${DIB_OPENSUSE_PATTERNS} python zypper sudo # Put in a dummy /etc/resolv.conf over the temporary one we used # to bootstrap. systemd has a bug/feature [1] that it will assume # you want systemd-networkd as the network manager and create a # broken symlink to /run/... if the base image doesn't have one. # This broken link confuses things like dhclient. # [1] https://bugzilla.redhat.com/show_bug.cgi?id=1197204 echo -e "# This file intentionally left blank\n" | \ sudo tee $TARGET_ROOT/etc/resolv.conf # set the most reliable UTF-8 locale echo -e 'LANG="en_US.UTF-8"' | \ sudo tee $TARGET_ROOT/etc/locale.conf # default to UTC sudo chroot $TARGET_ROOT ln -sf /usr/share/zoneinfo/UTC \ /etc/localtime # RPM doesn't know whether files have been changed since install # At this point though, we know for certain that we have changed no # config files, so anything marked .rpmnew is just a bug. for newfile in $(sudo find $TARGET_ROOT -type f -name '*rpmnew') ; do sudo mv $newfile $(echo $newfile | sed 's/.rpmnew$//') done # Unmounting of all the mount points is handled by the cleanup EXIT # handler so there is nothing else to do here diskimage-builder-2.11.0/diskimage_builder/elements/zypper/000077500000000000000000000000001324273057700237765ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/zypper/README.rst000066400000000000000000000011341324273057700254640ustar00rootroot00000000000000====== zypper ====== This element provides some customizations for zypper based distributions like SLES and openSUSE. It works in a very similar way as the yum element does for yum based distributions. Zypper is reconfigured so that it keeps downloaded packages cached outside of the build chroot so that they can be reused by subsequent image builds. The cache increases image building speed when building multiple images, especially on slow connections. This is more effective than using an HTTP proxy for caching packages since the download servers will often redirect clients to different mirrors. diskimage-builder-2.11.0/diskimage_builder/elements/zypper/bin/000077500000000000000000000000001324273057700245465ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/zypper/bin/install-packages000077500000000000000000000040731324273057700277220ustar00rootroot00000000000000#!/bin/bash # Copyright 2013 SUSE Linux Products GmbH # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail MAP_ELEMENT= ACTION=install ACTION_ARGS="--no-recommends" SCRIPTNAME=$(basename $0) function show_options () { echo "Usage: $SCRIPTNAME [package ...]" echo echo "Options:" echo " -u -- update all packages" echo " -e -- erase/remove packages" echo " -d -- download the packages only" echo " -m -- use custom element package map (Example: -m nova)" exit 0 } function run_zypper() { # TODO: Uncertain if this can ever block wanting input from user zypper --non-interactive --gpg-auto-import-keys "$@" } TEMP=$(getopt -o hudem: -n $SCRIPTNAME -- "$@") if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi eval set -- "$TEMP" while true ; do case "$1" in -u) run_zypper dist-upgrade --no-recommends; exit 0;; -e) ACTION="remove"; ACTION_ARGS="--clean-deps"; shift;; -d) ACTION_ARGS="${ACTION_ARGS} --download-only"; shift;; -m) MAP_ELEMENT=$2; shift 2;; -h) show_options;; --) shift; break ;; *) echo "Error: unsupported option $1."; exit 1;; esac done PKGS=$@ if [ -n "$MAP_ELEMENT" ]; then if ! PKGS=$(pkg-map --element $MAP_ELEMENT $@); then echo "bin/pkg-map error. $PKGS" exit 1 fi fi if [ -z "${PKGS}" ]; then echo "Not running install-packages $ACTION with empty packages list" else run_zypper $ACTION $ACTION_ARGS $PKGS fi diskimage-builder-2.11.0/diskimage_builder/elements/zypper/element-deps000066400000000000000000000000141324273057700262760ustar00rootroot00000000000000install-bin diskimage-builder-2.11.0/diskimage_builder/elements/zypper/install.d/000077500000000000000000000000001324273057700256665ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/zypper/install.d/01-login-defs000077500000000000000000000002441324273057700300610ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail sed -i -e "s,^USERGROUPS_ENAB.*$,USERGROUPS_ENAB yes," /etc/login.defs diskimage-builder-2.11.0/diskimage_builder/elements/zypper/post-install.d/000077500000000000000000000000001324273057700266515ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/zypper/post-install.d/10-mkinitrd000077500000000000000000000013161324273057700306370ustar00rootroot00000000000000#!/bin/bash # The base image created by kiwi does not contain an initrd so create one # here. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # This is to tell mkinitrd to include the right tools for the root filesystem # that will actually be used for the final image. This is likely something # different than what the chroot is currently on (which might currently be a # tmpfs even). echo "rootfstype=${DIB_ROOT_FSTYPE}" > /etc/sysconfig/initrd # ToDo: This it not really clear to me. # Let's see which error occurs. # openSuse mkinitrd requires a valid root device be in fstab. ##sed -i 's/vda1/sda1/' /etc/fstab mkinitrd -A -B # And cleanup again rm /etc/sysconfig/initrd diskimage-builder-2.11.0/diskimage_builder/elements/zypper/post-install.d/98-zypper-clean-cache000077500000000000000000000002261324273057700325070ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # clean all locally cached metadata sudo zypper clean -a diskimage-builder-2.11.0/diskimage_builder/elements/zypper/post-install.d/99-zypper-no-keep-packages000077500000000000000000000002751324273057700335030ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # Disable caching packages for all repositories sudo zypper modifyrepo --all --no-keep-packages diskimage-builder-2.11.0/diskimage_builder/elements/zypper/pre-install.d/000077500000000000000000000000001324273057700264525ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/zypper/pre-install.d/01-zypper-keep-packages000077500000000000000000000002641324273057700326470ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail # Enable caching packages for all repositories zypper modifyrepo --all --keep-packages diskimage-builder-2.11.0/diskimage_builder/elements/zypper/root.d/000077500000000000000000000000001324273057700252035ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/elements/zypper/root.d/50-zypper-cache000077500000000000000000000004511324273057700277450ustar00rootroot00000000000000#!/bin/bash if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu set -o pipefail [ -n "$TARGET_ROOT" ] ZYPPER_CACHE_DIR=$DIB_IMAGE_CACHE/zypper mkdir -p $ZYPPER_CACHE_DIR sudo mkdir -p $TMP_MOUNT_PATH/var/cache/zypp sudo mount --bind $ZYPPER_CACHE_DIR $TMP_MOUNT_PATH/var/cache/zypp diskimage-builder-2.11.0/diskimage_builder/lib/000077500000000000000000000000001324273057700213775ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/lib/common-defaults000066400000000000000000000033311324273057700244170ustar00rootroot00000000000000#!/bin/bash # Copyright 2012 Hewlett-Packard Development Company, L.P. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. if [ -f /etc/debian_version ]; then _ARCH=$(dpkg --print-architecture) else _ARCH=$(uname -m) # Standardise Architecture names case $_ARCH in "x86_64") _ARCH="amd64" ;; "armv"*) _ARCH="armhf" ;; "aarch64") _ARCH="arm64" ;; esac fi ARCH=${ARCH:-$_ARCH} export ARCH export DIB_CHECKSUM=${DIB_CHECKSUM:-0} export DIB_NO_TMPFS=${DIB_NO_TMPFS:-0} export DIB_MIN_TMPFS=${DIB_MIN_TMPFS:-2} # Set via the CLI normally. # IMAGE_ELEMENT= ELEMENTS_PATH=${ELEMENTS_PATH:+"$ELEMENTS_PATH:$_BASE_ELEMENT_DIR"} export ELEMENTS_PATH=${ELEMENTS_PATH:-$_BASE_ELEMENT_DIR} export DIB_OFFLINE=${DIB_OFFLINE:-''} # /sbin and friends aren't automatically included in $PATH when running sudo # on Fedora/RHEL/SUSE Linux which breaks various pieces of this script in # exciting ways, so inject them into the current $PATH if they're not there. if ! [[ ":$PATH" =~ ":/sbin" ]] ; then export PATH=$PATH:/sbin fi if ! [[ ":$PATH" =~ ":/usr/sbin" ]] ; then export PATH=$PATH:/usr/sbin fi if ! [[ ":$PATH" =~ ":/usr/local/sbin" ]] ; then export PATH=$PATH:/usr/local/sbin fi diskimage-builder-2.11.0/diskimage_builder/lib/common-functions000066400000000000000000000347351324273057700246340ustar00rootroot00000000000000#!/bin/bash # Copyright 2012 Hewlett-Packard Development Company, L.P. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # This is the "internal" verison of dib-run-parts. Note the currently # this is copied by the dib-run-parts element into the chroot. If you # modify this, be aware that it needs to run in both contexts. DIB_RUN_PARTS=${_LIB}/dib-run-parts function tmpfs_check() { local echo_message=${1:-1} [ "$DIB_NO_TMPFS" == "0" ] || return 1 [ -r /proc/meminfo ] || return 1 total_kB=$(awk '/^MemTotal/ { print $2 }' /proc/meminfo) # tmpfs uses by default 50% of the available RAM, so the RAM should be at least # the double of the minimum tmpfs size required RAM_NEEDED=$(($DIB_MIN_TMPFS * 2)) [ $total_kB -lt $(($RAM_NEEDED*1024*1024)) ] || return 0 if [ $echo_message == '1' ]; then echo "WARNING: Not enough RAM to use tmpfs for build. Using ${TMP_DIR:-/tmp}. ($total_kB < ${RAM_NEEDED}G)" fi return 1 } function mk_build_dir () { TMP_BUILD_DIR=$(mktemp -t -d --tmpdir=${TMP_DIR:-/tmp} dib_build.XXXXXXXX) TMP_IMAGE_DIR=$(mktemp -t -d --tmpdir=${TMP_DIR:-/tmp} dib_image.XXXXXXXX) [ $? -eq 0 ] || die "Failed to create tmp directory" export TMP_BUILD_DIR if tmpfs_check ; then sudo mount -t tmpfs tmpfs $TMP_BUILD_DIR sudo mount -t tmpfs tmpfs $TMP_IMAGE_DIR sudo chown $(id -u):$(id -g) $TMP_BUILD_DIR $TMP_IMAGE_DIR fi trap trap_cleanup EXIT echo Building in $TMP_BUILD_DIR export TMP_IMAGE_DIR export OUT_IMAGE_PATH=$TMP_IMAGE_PATH export TMP_HOOKS_PATH=$TMP_BUILD_DIR/hooks } function finish_image () { if [ -f $1 -a ${OVERWRITE_OLD_IMAGE:-0} -eq 0 ]; then old_image="${1%.*}"-$(date +%Y.%m.%d-%H.%M.%S).${1##*.} echo "Old image found. Renaming it to $old_image" mv "$1" "$old_image" if [ -f "$1.md5" ]; then mv "$1.md5" "$old_image.md5" fi if [ -f "$1.sha256" ]; then mv "$1.sha256" "$old_image.sha256" fi fi mv $OUT_IMAGE_PATH $1 if [ "$DIB_CHECKSUM" == "1" ]; then # NOTE(pabelanger): Read image into memory once and generate both checksum # files. md5sum $1 > $1.md5 & sha256sum $1 > $1.sha256 & wait fi echo "Image file $1 created..." } function save_image () { finish_image $1 } function copy_hooks_not_overwrite () { _DIR=$(basename $1) test -d $TMP_HOOKS_PATH/$_DIR || mkdir $TMP_HOOKS_PATH/$_DIR for _HOOK in $(ls $1); do if [ ! -f $TMP_HOOKS_PATH/$_DIR/$_HOOK ]; then echo "Copying hooks $1/$_HOOK" cp -t $TMP_HOOKS_PATH/$_DIR -a $1/$_HOOK else echo "There is a duplicated hook in your elements: $_ELEMENT/$_DIR/$_HOOK" exit 1 fi done } function generate_hooks () { local dir local file eval declare -A image_elements=($(get_image_element_array)) mkdir -p $TMP_HOOKS_PATH for i in "${!image_elements[@]}"; do local element=$i local element_dir=${image_elements[$i]} for dir in $(find $element_dir \ -follow -mindepth 1 -maxdepth 1 \ -type d -not -name tests); do copy_hooks_not_overwrite $dir done for file in $(find $element_dir \ -follow -maxdepth 1 -type f); do cp -t $TMP_HOOKS_PATH -a $file done done } # Call the supplied break-in routine if the named point is listed in the break # list. # $1 the break point. # $2.. what to call if a break is needed function check_break () { if echo "${break:-}" | egrep -e "(,|^)$1(,|$)" -q; then echo "Starting debug shell. Exit to resume building." >&2 echo At stage $1 >&2 shift "$@" echo "Resuming" >&2 fi } # Check that a real element has been chosen (prevents foot-guns) function check_element () { [ -d $TMP_HOOKS_PATH ] || generate_hooks } # Run a hook, looking for a regex in its stdout, and eval the matched lines. # $1 is the hook to run # $2 is the regex to look for function eval_run_d () { local run_output=$(mktemp) trap "rm -f $run_output; check_break after-error ${break_cmd:-bash}" ERR run_d $1 $run_output if grep -q "$2" $run_output; then local temp=$(grep "$2" $run_output) eval "$temp" fi rm $run_output trap - ERR } # Get any process that appears to be running in $TMP_BUILD_DIR function _get_chroot_processes () { # Deselect kernel threads, and use a python script to avoid # forking lots and lots of readlink / grep processes on a busy # system. ps --ppid 2 -p 2 --deselect -o pid= | xargs ${DIB_PYTHON_EXEC:-python} -c ' import os import sys for pid in sys.argv[2:]: try: root = os.readlink("/proc/%s/root" % pid) except: continue if sys.argv[1] in root: print("%s" % pid) ' $TMP_BUILD_DIR } function kill_chroot_processes () { local xtrace xtrace=$(set +o | grep xtrace) set +o xtrace local pidname if [ -z "${1}" ]; then echo "ERROR: no chroot directory specified" exit 1 fi for pid in $(_get_chroot_processes); do # If there are open files from the chroot, just kill the process using # these files. This is racy, but good enough pidname=$(cat $piddir/comm 2>/dev/null || echo "unknown") echo "Killing chroot process: '${pidname}($pid)'" sudo kill $pid done $xtrace } function cleanup_build_dir () { if ! timeout 10 sh -c " while ! sudo rm -rf $TMP_BUILD_DIR/built; do sleep 1; done"; then echo "ERROR: unable to cleanly remove $TMP_BUILD_DIR/built" exit 1 fi sudo rm -rf $TMP_BUILD_DIR/mnt kill_chroot_processes $TMP_BUILD_DIR if tmpfs_check 0; then # If kill_chroot_processes did not succeed then we have to wait for # init to reap the orphaned chroot processes if ! timeout 120 sh -c "while ! sudo umount -f $TMP_BUILD_DIR; do sleep 1; done"; then echo "ERROR: failed to umount the $TMP_BUILD_DIR tmpfs mount point" exit 1 fi fi rm -rf --one-file-system $TMP_BUILD_DIR } function cleanup_image_dir () { kill_chroot_processes $TMP_IMAGE_DIR if tmpfs_check 0; then if ! timeout 120 sh -c "while ! sudo umount -f $TMP_IMAGE_DIR; do sleep 1; done"; then echo "ERROR: failed to umount the $TMP_IMAGE_DIR tmpfs mount point" exit 1 fi fi rm -rf --one-file-system $TMP_IMAGE_DIR } # Run a directory of hooks outside the target (that is, no chrooting). function run_d() { check_element check_break before-$1 ${break_cmd:-bash} if [ -d ${TMP_HOOKS_PATH}/$1.d ] ; then echo "Running hooks from ${TMP_HOOKS_PATH}/$1.d" if [ -n "$2" ]; then ${DIB_RUN_PARTS} ${TMP_HOOKS_PATH}/$1.d | tee $2 if [[ ${PIPESTATUS[0]} != 0 ]]; then return 1 fi else ${DIB_RUN_PARTS} ${TMP_HOOKS_PATH}/$1.d fi fi check_break after-$1 bash } function arg_to_elements() { for arg do IMAGE_ELEMENT="$IMAGE_ELEMENT $arg" ; done if [ "$SKIP_BASE" != "1" ]; then IMAGE_ELEMENT="base $IMAGE_ELEMENT" fi if [ "$IS_RAMDISK" == "1" ]; then IMAGE_ELEMENT="$RAMDISK_ELEMENT $IMAGE_ELEMENT" fi echo "Building elements: $IMAGE_ELEMENT" export IMAGE_ELEMENT # element-info will output bash code to create # * IMAGE_ELEMENT # legacy list of elements # # * IMAGE_ELEMENT_YAML # YAML dictionary with key=element, value=path # # import os # import yaml # yaml.load(os.getenv('IMAGE_ELEMENT_YAML') # # * function get_image_element_array # Function to create Bash associative-array. Since bash can not # export array variables, we provide a function to populate the # variables. # # # we need the eval, it expands the string for the array create # eval declare -A image_elements=($(get_image_element_array)) # for i in ${!image_elements[@]}; do # element=$i # path=${image_elements[$i] # done elinfo_out="$(element-info --env $IMAGE_ELEMENT)" if [ $? -ne 0 ]; then echo "ERROR: element-info failed to expand elements." exit 1 fi eval "$elinfo_out" echo "Expanded element dependencies to: $IMAGE_ELEMENT" } function create_base () { mkdir $TMP_BUILD_DIR/mnt export TMP_MOUNT_PATH=$TMP_BUILD_DIR/mnt # Copy data in to the root. TARGET_ROOT=$TMP_MOUNT_PATH run_d root if [ -z "$(ls $TMP_MOUNT_PATH | grep -v '^lost+found\|tmp$')" ] ; then # No root element copied in. Note the test above allows # root.d elements to put things in /tmp echo "Failed to deploy the root element." exit 1 fi # Configure Image # Setup resolv.conf so we can chroot to install some packages if [ -L $TMP_MOUNT_PATH/etc/resolv.conf ] || [ -f $TMP_MOUNT_PATH/etc/resolv.conf ] ; then sudo mv $TMP_MOUNT_PATH/etc/resolv.conf $TMP_MOUNT_PATH/etc/resolv.conf.ORIG fi # Recreate resolv.conf sudo touch $TMP_MOUNT_PATH/etc/resolv.conf sudo chmod 777 $TMP_MOUNT_PATH/etc/resolv.conf # use system configured resolv.conf if available to support internal proxy resolving if [ -e /etc/resolv.conf ]; then cat /etc/resolv.conf > $TMP_MOUNT_PATH/etc/resolv.conf else echo nameserver 8.8.8.8 > $TMP_MOUNT_PATH/etc/resolv.conf fi mount_proc_dev_sys } # Get mount options for mounting /dev/pts # Kernel commit https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=eedf265aa003b4781de24cfed40a655a664457e6 # introduced in v4.7 allows multiple instances of devpts. However, # some distributions are running older kernels so we need to take # care on what options we use to mount a new instance of devpts # filesystem since it's not completely independent. The best thing # to do is to simply use the existing mount options. function mount_dev_pts_options() { echo "-o $(findmnt --first-only /dev/pts --noheadings --output OPTIONS)" } function mount_proc_dev_sys () { # supporting kernel file systems sudo mount -t proc none $TMP_MOUNT_PATH/proc sudo mount --bind /dev $TMP_MOUNT_PATH/dev sudo mount -t devpts $(mount_dev_pts_options) devpts $TMP_MOUNT_PATH/dev/pts sudo mount -t sysfs none $TMP_MOUNT_PATH/sys } # Recursively unmount directories under a given directory DIR # usage: # unmount_dir DIR function unmount_dir { local dir="$1" local real_dir local mnts local split_mounts local found_mountpoint if [ ! -d $dir ]; then echo "*** $dir is not a directory" return 0 fi # get rid of any symlink elements in the incoming path, because # /proc/mounts is the real path real_dir=$(readlink -e $dir) # populate the exported mountpoints IFS='|' read -ra split_mounts <<< "$DIB_MOUNTPOINTS" # note the "/" on real_dir ... we are just looking for things # mounted *underneath* this directory. mnts=$(awk '{print $2}' < /proc/mounts | grep "^$real_dir/" | sort -r) for m in $mnts; do # check if suffix is in array found_mountpoint=false for mountpoint in "${split_mounts[@]}"; do if [[ "$mountpoint" != "/" ]]; then if [[ "$m" == *$mountpoint ]]; then echo "Mountpoint $m managed by block device; skipping" found_mountpoint=true break fi fi done if [ $found_mountpoint == false ]; then # unmount the directory as it is not managed by block device echo "Unmount $m" sudo umount -fl $m || true fi done } # Create YAML config file for the block device layer # The order here is: use the one the user provides - if there is # none provided, fall back to the possible one element which # defines a fallback configuration. # Parameters: # - name of the to be created config file function block_device_create_config_file { # nosiy; we manually trace local xtrace xtrace=$(set +o | grep xtrace) set +o xtrace local config_yaml="$1" if [[ ${DIB_BLOCK_DEVICE_CONFIG:-} == file://* ]]; then cp $(echo ${DIB_BLOCK_DEVICE_CONFIG} | cut -c 8-) ${config_yaml} echo "Using file-based block-device config: ${DIB_BLOCK_DEVICE_CONFIG}" return fi if [ -n "${DIB_BLOCK_DEVICE_CONFIG:-}" ]; then printf "%s" "${DIB_BLOCK_DEVICE_CONFIG}" >${config_yaml} echo "User specified block-device config from DIB_BLOCK_DEVICE_CONFIG" return fi # Search the elements for a matching block-device config. # XXX: first match wins? echo "Searching elements for block-device[-${ARCH}].yaml ..." eval declare -A image_elements=($(get_image_element_array)) for i in ${!image_elements[@]}; do local cfg # look for arch specific version first, then default if [[ "ppc64le ppc64el" =~ $ARCH ]] ; then # NOTE(tonyb): ppc64el and ppc64le are the same archttechture, it's # just different distro's have different names. So if we're either # of them pick the block-device-ppc64el.yaml file cfg=${image_elements[$i]}/block-device-ppc64el.yaml else cfg=${image_elements[$i]}/block-device-${ARCH}.yaml fi if [ -e ${cfg} ]; then cp ${cfg} ${config_yaml} echo "Using block-device config: ${cfg}" return else cfg=${image_elements[$i]}/block-device-default.yaml if [ -e ${cfg} ]; then cp ${cfg} ${config_yaml} echo "Using block-device config: ${cfg}" return fi fi done echo "... done" # how did this get here? if [ -e ${config_yaml} ]; then die "${config_yaml} exists?" fi echo "Using default block-device fallback config" # If no config is there (until now) use the default config cat >${config_yaml} <&2 output_prefix() { printf "%s " "${name}" >&2 } output () { output_prefix echo $* >&2 } output_printf () { local FORMAT="$1" shift output_prefix printf "${FORMAT}" $@ >&2 } # source the environment files from environment.d # arg : target_dir source_environment() { local dir=$target_dir/../environment.d local env_files local xtrace if [ -d ${dir} ] ; then env_files=$(find ${dir} -maxdepth 1 -xtype f | \ grep -E "/[0-9A-Za-z_\.-]+$" | \ LANG=C sort -n) for env_file in $env_files ; do output "Sourcing environment file ${env_file}" # Set tracing as we import these environment files; it's # nice to see the definitions in the logs xtrace=$(set +o | grep xtrace) set -o xtrace source $env_file $xtrace done fi } if [ $# -lt 1 ] ; then usage fi if [ "$1" == "--list" ] ; then show_list="1" shift fi target_dir="${1:-}" if ! [ -d "$target_dir" ] ; then output "Scripts directory [$target_dir] must exist and be a directory" usage fi # We specifically only want to sort *by the numbers*. # Lexical sorting is not guaranteed, and identical numbers may be # parallelized later # Note: -maxdepth 1 ensures only files in the target directory (but not # subdirectories) are run, which is the way run-parts behaves. targets=$(find $target_dir -maxdepth 1 -xtype f -executable -printf '%f\n' | grep -E "$allowed_regex" | LANG=C sort -n || echo "") if [ "$show_list" == "1" ] ; then for target in $targets ; do echo "${target_dir}/${target}" done exit 0 fi PROFILE_DIR=$(mktemp -d --tmpdir profiledir.XXXXXX) # note, run this in a sub-shell so we don't pollute our # own environment with source_environment ( source_environment for target in $targets ; do output "Running $target_dir/$target" target_tag=${target//\//_} date +%s.%N > $PROFILE_DIR/start_$target_tag $target_dir/$target target_tag=${target//\//_} date +%s.%N > $PROFILE_DIR/stop_$target_tag output "$target completed" done ) output "----------------------- PROFILING -----------------------" output "" output "Target: $(basename $target_dir)" output "" output_printf "%-40s %9s\n" Script Seconds output_printf "%-40s %9s\n" --------------------------------------- ---------- output "" pushd $PROFILE_DIR > /dev/null for target in $(find . -name 'start_*' -printf '%f\n' | env LC_ALL=C sort -n) ; do stop_file=stop_${target##start_} start_seconds=$(cat $target) stop_seconds=$(cat $stop_file) duration=$(echo - | awk "{ print $stop_seconds - $start_seconds }") LC_NUMERIC=C LC_ALL=C output_printf "%-40s %10.3f\n" ${target##start_} $duration done popd > /dev/null rm -rf $PROFILE_DIR output "" output "--------------------- END PROFILING ---------------------" diskimage-builder-2.11.0/diskimage_builder/lib/die000066400000000000000000000014341324273057700220650ustar00rootroot00000000000000#!/bin/bash # Copyright 2012 Hewlett-Packard Development Company, L.P. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # Prints "message" and exits # Usage: die "message" function die { local exitcode=$? set +o xtrace echo $@ exit $exitcode } export -f die diskimage-builder-2.11.0/diskimage_builder/lib/disk-image-create000066400000000000000000000513231324273057700246010ustar00rootroot00000000000000#!/bin/bash # # Copyright 2012 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. set -eE # Set/override locale. This ensures consistency in sorting etc. We # need to choose a lowest-common denominator locale, as this is # applied when running in the building chroot too (maybe a bug and we # should prune this?). Thus "C" --centOS 7 doesn't include C.utf-8 # (fedora does, centos 8 probably will). Note: LC_ALL to really # override this; it overrides LANG and all other LC_ vars export LC_ALL=C # Store our initial environment and command line args for later export DIB_ARGS="$@" export DIB_ENV=$(export | grep ' DIB_.*=') SCRIPTNAME=$(basename $0) if [ -z "$_LIB" ]; then echo "_LIB not set!" exit 1 fi _BASE_ELEMENT_DIR=$(${DIB_PYTHON_EXEC:-python} -c ' import diskimage_builder.paths diskimage_builder.paths.show_path("elements")') source $_LIB/die IS_RAMDISK=0 if [ "$SCRIPTNAME" == "ramdisk-image-create" ]; then IS_RAMDISK=1 fi function show_options () { echo "Usage: ${SCRIPTNAME} [OPTION]... [ELEMENT]..." echo echo "Options:" echo " -a i386|amd64|armhf|arm64 -- set the architecture of the image(default amd64)" echo " -o imagename -- set the imagename of the output image file(default image)" echo " -t qcow2,tar,tgz,squashfs,vhd,docker,aci,raw -- set the image types of the output image files (default qcow2)" echo " File types should be comma separated. VHD outputting requires the vhd-util" echo " executable be in your PATH. ACI outputting requires the ACI_MANIFEST " echo " environment variable be a path to a manifest file." echo " -x -- turn on tracing (use -x -x for very detailed tracing)." echo " -u -- uncompressed; do not compress the image - larger but faster" echo " -c -- clear environment before starting work" echo " --logfile -- save run output to given logfile" echo " --checksum -- generate MD5 and SHA256 checksum files for the created image" echo " --image-size size -- image size in GB for the created image" echo " --image-cache directory -- location for cached images(default ~/.cache/image-create)" echo " --max-online-resize size -- max number of filesystem blocks to support when resizing." echo " Useful if you want a really large root partition when the image is deployed." echo " Using a very large value may run into a known bug in resize2fs." echo " Setting the value to 274877906944 will get you a 1PB root file system." echo " Making this value unnecessarily large will consume extra disk space " echo " on the root partition with extra file system inodes." echo " --min-tmpfs size -- minimum size in GB needed in tmpfs to build the image" echo " --mkfs-options -- option flags to be passed directly to mkfs." echo " Options should be passed as a single string value." echo " --no-tmpfs -- do not use tmpfs to speed image build" echo " --offline -- do not update cached resources" echo " --qemu-img-options -- option flags to be passed directly to qemu-img." echo " Options need to be comma separated, and follow the key=value pattern." echo " --root-label label -- label for the root filesystem. Defaults to 'cloudimg-rootfs'." echo " --ramdisk-element -- specify the main element to be used for building ramdisks." echo " Defaults to 'ramdisk'. Should be set to 'dracut-ramdisk' for platforms such" echo " as RHEL and CentOS that do not package busybox." echo " --install-type -- specify the default installation type. Defaults to 'source'. Set to 'package' to use package based installations by default." echo " --docker-target -- specify the repo and tag to use if the output type is docker. Defaults to the value of output imagename" if [ "$IS_RAMDISK" == "0" ]; then echo " -n skip the default inclusion of the 'base' element" echo " -p package[,p2...] [-p p3] -- extra packages to install in the image. Runs once, after 'install.d' phase. Can be specified mulitple times" fi echo " -h|--help -- display this help and exit" echo " --version -- display version and exit" echo echo "Environment Variables:" echo " (this is not a complete list)" echo echo " * ELEMENTS_PATH: specify external locations for the elements. As for \$PATH" echo " * DIB_NO_TIMESTAMP: no timestamp prefix on output. Useful if capturing output" echo " * DIB_QUIET: do not output log output to stdout. See --logfile" echo echo "NOTE: At least one distribution root element must be specified." echo echo "NOTE: If using the VHD output format you need to have a patched version of vhd-util installed for the image" echo " to be bootable. The patch is available here: https://github.com/emonty/vhd-util/blob/master/debian/patches/citrix" echo " and a PPA with the patched tool is available here: https://launchpad.net/~openstack-ci-core/+archive/ubuntu/vhd-util" echo echo "Examples:" if [ "$IS_RAMDISK" == "0" ]; then echo " ${SCRIPTNAME} -a amd64 -o ubuntu-amd64 vm ubuntu" echo " export ELEMENTS_PATH=~/source/tripleo-image-elements/elements" echo " ${SCRIPTNAME} -a amd64 -o fedora-amd64-heat-cfntools vm fedora heat-cfntools" else echo " ${SCRIPTNAME} -a amd64 -o fedora-deploy deploy fedora" echo " ${SCRIPTNAME} -a amd64 -o ubuntu-ramdisk ramdisk ubuntu" fi } function show_version() { ${DIB_PYTHON_EXEC:-python} -c "from diskimage_builder import version; print(version.version_info.version_string())" } DIB_DEBUG_TRACE=${DIB_DEBUG_TRACE:-0} INSTALL_PACKAGES="" IMAGE_TYPES=("qcow2") COMPRESS_IMAGE="true" ROOT_LABEL="" DIB_DEFAULT_INSTALLTYPE=${DIB_DEFAULT_INSTALLTYPE:-"source"} MKFS_OPTS="" ACI_MANIFEST=${ACI_MANIFEST:-} DOCKER_TARGET="" LOGFILE="" TEMP=`getopt -o a:ho:t:xucnp: -l checksum,no-tmpfs,offline,help,version,min-tmpfs:,image-size:,image-cache:,max-online-resize:,mkfs-options:,qemu-img-options:,ramdisk-element:,root-label:,install-type:,docker-target:,logfile: -n $SCRIPTNAME -- "$@"` if [ $? -ne 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi # Note the quotes around `$TEMP': they are essential! eval set -- "$TEMP" while true ; do case "$1" in -a) export ARCH=$2; shift 2 ;; -o) export IMAGE_NAME=$2; shift 2 ;; -t) IFS="," read -a IMAGE_TYPES <<< "$2"; export IMAGE_TYPES ; shift 2 ;; -h|--help) show_options; exit 0;; --version) show_version; exit 0;; -x) shift; DIB_DEBUG_TRACE=$(( $DIB_DEBUG_TRACE + 1 ));; -u) shift; export COMPRESS_IMAGE="";; -c) shift ; export CLEAR_ENV=1;; -n) shift; export SKIP_BASE="1";; -p) IFS="," read -a _INSTALL_PACKAGES <<< "$2"; export INSTALL_PACKAGES=( ${INSTALL_PACKAGES[@]} ${_INSTALL_PACKAGES[@]} ) ; shift 2 ;; --checksum) shift; export DIB_CHECKSUM=1;; --image-size) export DIB_IMAGE_SIZE=$2; shift 2;; --image-cache) export DIB_IMAGE_CACHE=$2; shift 2;; --max-online-resize) export MAX_ONLINE_RESIZE=$2; shift 2;; --mkfs-options) MKFS_OPTS=$2; shift 2;; --min-tmpfs) export DIB_MIN_TMPFS=$2; shift 2;; --no-tmpfs) shift; export DIB_NO_TMPFS=1;; --offline) shift; export DIB_OFFLINE=1;; --qemu-img-options) QEMU_IMG_OPTIONS=$2; shift 2;; --root-label) ROOT_LABEL=$2; shift 2;; --ramdisk-element) RAMDISK_ELEMENT=$2; shift 2;; --install-type) DIB_DEFAULT_INSTALLTYPE=$2; shift 2;; --docker-target) export DOCKER_TARGET=$2; shift 2 ;; --logfile) export LOGFILE=$2; shift 2 ;; --) shift ; break ;; *) echo "Internal error!" ; exit 1 ;; esac done export DIB_DEBUG_TRACE # TODO: namespace this under ~/.cache/dib/ for consistency export DIB_IMAGE_CACHE=${DIB_IMAGE_CACHE:-~/.cache/image-create} mkdir -p $DIB_IMAGE_CACHE # We have a couple of critical sections (touching parts of the host # system or download images to common cache) that we use flock around. # Use this directory for lockfiles. export DIB_LOCKFILES=${DIB_LOCKFILES:-~/.cache/dib/lockfiles} mkdir -p $DIB_LOCKFILES if [ "$CLEAR_ENV" = "1" -a "$HOME" != "" ]; then echo "Re-execing to clear environment." echo "(note this will prevent much of the local_config element from working)" exec -c $0 "$@" fi # We send stdout & stderr through "outfilter" which does timestamping, # basic filtering and log file output. _TS_FLAG="" if [[ "${DIB_NO_TIMESTAMP:-0}" -eq 1 ]]; then _TS_FLAG="--no-timestamp" fi _QUIET_FLAG="-v" if [[ "${DIB_QUIET:-0}" -eq 1 ]]; then _QUIET_FLAG="" fi _LOGFILE_FLAG="" if [[ -n "${LOGFILE}" ]]; then echo "Output logs going to: ${LOGFILE}" _LOGFILE_FLAG="-o ${LOGFILE}" fi exec 1> >( ${DIB_PYTHON_EXEC:-python} $_LIB/outfilter.py ${_TS_FLAG} ${_QUIET_FLAG} ${_LOGFILE_FLAG} ) 2>&1 # Display the current file/function/line in the debug output function _ps4 { IFS=" " called=($(caller 0)) local f=$(readlink -f ${called[2]}) printf "%-80s " "$f:${called[1]}:${called[0]}" } export -f _ps4 export PS4='+ $(_ps4): ' source $_LIB/img-defaults source $_LIB/common-functions source $_LIB/img-functions if [ "$IS_RAMDISK" == "1" ]; then source $_LIB/ramdisk-defaults source $_LIB/ramdisk-functions fi echo "diskimage-builder version $(show_version)" # If no elements are specified theres no way we can succeed if [ -z "$*" ]; then echo "ERROR: At least one distribution root element must be specified" exit 1 fi arg_to_elements "$@" # start tracing after most boilerplate if [ ${DIB_DEBUG_TRACE} -gt 0 ]; then set -x fi if [ "${#IMAGE_TYPES[@]}" = "1" ]; then export IMAGE_NAME=${IMAGE_NAME%%\.${IMAGE_TYPES[0]}} fi # Check for required tools early on for X in ${!IMAGE_TYPES[@]}; do case "${IMAGE_TYPES[$X]}" in qcow2) if ! type qemu-img > /dev/null 2>&1; then echo "qcow2 output format specified but qemu-img executable not found." exit 1 fi ;; tgz) # Force tar to be created. IMAGE_TYPES+=('tar') ;; vhd) if ! type vhd-util > /dev/null 2>&1; then echo "vhd output format specified but no vhd-util executable found." exit 1 fi ;; squashfs) if ! type mksquashfs > /dev/null 2>&1; then echo "squashfs output format specified but no mksquashfs executable found." exit 1 fi ;; docker) if ! type docker > /dev/null 2>&1; then echo "docker output format specified but no docker executable found." exit 1 fi if [ -z "$DOCKER_TARGET" ]; then echo "Please set --docker-target." exit 1 fi ;; esac done # NOTE: fstrim is on most all recent systems. It is provided by the util-linux # package. if ! type fstrim > /dev/null 2>&1; then echo "fstrim utility is not found. This is provided by util-linux package" echo "Please check your PATH variable is set correctly" exit 1 fi # xattr support cannot be relied upon with tmpfs builds # some kernels supoprt it, some don't if [[ -n "${GENTOO_PROFILE}" ]]; then if [[ "${GENTOO_PROFILE}" =~ "hardened" ]]; then echo 'disabling tmpfs for gentoo hardened build' export DIB_NO_TMPFS=1 fi fi mk_build_dir # Create the YAML file with the final and raw configuration for # the block device layer. mkdir -p ${TMP_BUILD_DIR}/block-device BLOCK_DEVICE_CONFIG_YAML=${TMP_BUILD_DIR}/block-device/config.yaml block_device_create_config_file "${BLOCK_DEVICE_CONFIG_YAML}" # Write out the parameter file DIB_BLOCK_DEVICE_PARAMS_YAML=${TMP_BUILD_DIR}/block-device/params.yaml export DIB_BLOCK_DEVICE_PARAMS_YAML cat >${DIB_BLOCK_DEVICE_PARAMS_YAML} < $TMP_HOOKS_PATH/environment.d/11-dib-install-type.bash run_d extra-data # Run pre-install scripts. These do things that prepare the chroot for package installs run_d_in_target pre-install # Call install scripts to pull in the software users want. run_d_in_target install do_extra_package_install run_d_in_target post-install # ensure we do not have a lost+found directory in the root folder # that could cause copy to fail (it will be created again later # when creating the file system, if it needs such directory) if [ -e "$TMP_BUILD_DIR/mnt/lost+found" ]; then sudo rm -rf "$TMP_BUILD_DIR/mnt/lost+found" fi # Free up /mnt unmount_image mv $TMP_BUILD_DIR/mnt $TMP_BUILD_DIR/built # save xtrace state, as we always want to turn it off to avoid # spamming the logs with du output below. xtrace=$(set +o | grep xtrace) # temp file for holding du output du_output=${TMP_BUILD_DIR}/du_output.tmp if [ -n "$DIB_IMAGE_SIZE" ]; then du_size=$(echo "$DIB_IMAGE_SIZE" | awk '{printf("%d\n",$1 * 1024 *1024)}') else set +o xtrace echo "Calculating image size (this may take a minute)..." sudo du -a -c -x ${TMP_BUILD_DIR}/built > ${du_output} # the last line is the total size from "-c". # scale this by 0.6 to create a slightly bigger image du_size=$(tail -n1 ${du_output} | cut -f1 | awk '{print int($1 / 0.6)}') $xtrace fi if [[ "${DIB_SHOW_IMAGE_USAGE:-0}" != 0 ]]; then set +o xtrace if [ ! -f "$du_output" ]; then sudo du -a -c -x ${TMP_BUILD_DIR}/built > ${du_output} fi du_output_show="sort -nr ${du_output} | numfmt --to=iec-i --padding=7 --suffix=B --field=1 --from-unit=1024" # by default show the 10MiB and greater files & directories -- a # dir with lots of little files will still show up, but this helps # signal:noise ratio if [[ ${DIB_SHOW_IMAGE_USAGE_FULL:-0} == 0 ]]; then # numfmt will start giving a decimal place when < 10MiB du_output_show+="| egrep 'MiB|GiB|TiB|PiB' | grep -v '\..MiB'" echo "=================================" echo "Image size report (files > 10MiB)" echo "=================================" else echo "=================" echo "Image size report" echo "=================" fi eval ${du_output_show} echo echo "===== end image size report =====" echo $xtrace fi rm -f ${du_output} if [ "$DIB_ROOT_FSTYPE" = "ext4" ] ; then # Very conservative to handle images being resized a lot # We set journal size to 64M so our journal is large enough when we # perform an FS resize. MKFS_OPTS="-i 4096 -J size=64 $MKFS_OPTS" # Grow the image size to account for the journal, only if the user # has not asked for a specific size. if [ -z "$DIB_IMAGE_SIZE" ]; then du_size=$(( $du_size + 65536 )) fi fi # Rounding down size so that is is a multiple of 64, works around a bug in # qemu-img that may occur when compressing raw images that aren't a multiple # of 64k. https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1180021 export DIB_IMAGE_SIZE=$(echo "$du_size" | awk ' { if ($1 % 64 != 0) print $1 + 64 - ( $1 % 64); else print $1; } ') if [ -n "$MAX_ONLINE_RESIZE" ]; then MKFS_OPTS="-E resize=$MAX_ONLINE_RESIZE $MKFS_OPTS" fi export TMP_IMAGE_DIR # Try the 'old fashioned' way calling the block device # phase. If this gives no result, use the configuration based approach: eval_run_d block-device "IMAGE_BLOCK_DEVICE=" if [ -z ${IMAGE_BLOCK_DEVICE} ] ; then # For compatibily reasons in addition to the YAML configuration # there is the need to handle the old environment variables. echo "image-size: ${DIB_IMAGE_SIZE}KiB" >> ${DIB_BLOCK_DEVICE_PARAMS_YAML} if [ -n "${MKFS_OPTS}" ] ; then echo "root-fs-opts: '${MKFS_OPTS}'" >> ${DIB_BLOCK_DEVICE_PARAMS_YAML} fi # After changeing the parameters, there is the need to # re-run dib-block-device init because some value might # change based on the new set parameters. dib-block-device init # values to dib-block-device: using the YAML config and dib-block-device create # This is the device (/dev/loopX). It's where to install the # bootloader. IMAGE_BLOCK_DEVICE=$(dib-block-device getval image-block-device) export IMAGE_BLOCK_DEVICE # Similar to above, but all mounted devices. This is handy for # some bootloaders that have multi-partition layouts and want to # copy things to different places other than just # IMAGE_BLOCK_DEVICE. "eval" this into an array as needed IMAGE_BLOCK_DEVICES=$(dib-block-device getval image-block-devices) export IMAGE_BLOCK_DEVICES # Write the fstab dib-block-device writefstab fi # XXX: needed? LOOPDEV=${IMAGE_BLOCK_DEVICE} # At this point, dib-block-device has created the raw image file # (IMAGE_BLOCK_DEVICE) and mounted all the partitions under # $TMP_BUILD_DIR/mnt for us. We can now copy into the final image. # 'mv' is not usable here - especially when a top level directory # has the same name as a mount point of a partition. If so, 'mv' # will complain: # mv: inter-device move failed: '...' to '...'; \ # unable to remove target: Device or resource busy # therefore a 'cp' and 'rm' approach is used. sudo cp -ra ${TMP_BUILD_DIR}/built/* $TMP_BUILD_DIR/mnt sudo rm -fr ${TMP_BUILD_DIR}/built/* mount_proc_dev_sys run_d_in_target finalise finalise_base for X in ${!IMAGE_TYPES[@]} ; do if [[ " tar aci " =~ "${IMAGE_TYPES[$X]}" ]]; then if [ "${IMAGE_TYPES[$X]}" = "aci" ]; then sudo tar -C ${TMP_BUILD_DIR}/mnt -cf $IMAGE_NAME.aci --exclude ./sys \ --exclude ./proc --xattrs --xattrs-include=\* \ --transform 's,^.,rootfs,S' . if [ -n "$ACI_MANIFEST" ]; then cp $ACI_MANIFEST ${TMP_BUILD_DIR}/manifest sudo tar -C ${TMP_BUILD_DIR} --append -f $IMAGE_NAME.aci manifest else echo "No ACI_MANIFEST specified. An ACI_MANIFEST must be specified for" echo " this image to be usable." fi else sudo tar -C ${TMP_BUILD_DIR}/mnt -cf $IMAGE_NAME.tar --exclude ./sys \ --exclude ./proc --xattrs --xattrs-include=\* . fi sudo chown $USER: $IMAGE_NAME.${IMAGE_TYPES[$X]} unset IMAGE_TYPES[$X] elif [ "${IMAGE_TYPES[$x]}" == "squashfs" ]; then sudo mksquashfs ${TMP_BUILD_DIR}/mnt $IMAGE_NAME.squash -comp xz \ -noappend -root-becomes ${TMP_BUILD_DIR}/mnt \ -wildcards -e "proc/*" -e "sys/*" -no-recovery elif [ "${IMAGE_TYPES[$X]}" == "docker" ]; then sudo tar -C ${TMP_BUILD_DIR}/mnt -cf - --exclude ./sys \ --exclude ./proc --xattrs --xattrs-include=\* . \ | sudo docker import - $DOCKER_TARGET unset IMAGE_TYPES[$X] fi done # Unmount and cleanup the /mnt and /build subdirectories, to save # space before converting the image to some other format. # XXX ? needed? export EXTRA_UNMOUNT="" unmount_image TMP_IMAGE_PATH=$(dib-block-device getval image-path) export TMP_IMAGE_PATH # remove all mounts dib-block-device umount dib-block-device cleanup cleanup_build_dir if [[ (! $IMAGE_ELEMENT =~ no-final-image) && "$IS_RAMDISK" == "0" ]]; then has_raw_type= for IMAGE_TYPE in ${IMAGE_TYPES[@]} ; do # We have to do raw last because it is destructive if [ "$IMAGE_TYPE" = "raw" ]; then has_raw_type=1 elif [ "$IMAGE_TYPE" != "squashfs" ]; then compress_and_save_image $IMAGE_NAME.$IMAGE_TYPE fi done if [ -n "$has_raw_type" ]; then IMAGE_TYPE="raw" compress_and_save_image $IMAGE_NAME.$IMAGE_TYPE fi fi # Remove the leftovers, i.e. the temporary image directory. cleanup_image_dir # All done! trap EXIT diskimage-builder-2.11.0/diskimage_builder/lib/img-defaults000066400000000000000000000015521324273057700237060ustar00rootroot00000000000000#!/bin/bash # Copyright 2012 Hewlett-Packard Development Company, L.P. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. source $_LIB/common-defaults # options for create-baremetal-image.sh export FS_TYPE=${FS_TYPE:-ext4} # Used to set the file extension only at this stage. export IMAGE_TYPE=${IMAGE_TYPE:-qcow2} export IMAGE_NAME=${IMAGE_NAME:-image} diskimage-builder-2.11.0/diskimage_builder/lib/img-functions000066400000000000000000000211531324273057700241060ustar00rootroot00000000000000#!/bin/bash # Copyright 2012 Hewlett-Packard Development Company, L.P. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. function unmount_image () { # Calling sync before helps ensure the mount isn't busy when you unmount it. # Previously observing having disk corruption issues; one possibility is # qemu-nbd not flushing dirty pages on disconnect? # https://bugs.launchpad.net/diskimage-builder/+bug/1214388 sync # unmount from the chroot # Don't use TMP_MOUNT_PATH here, it might not have been set. unmount_dir "$TMP_BUILD_DIR/mnt" if [ -n "$EXTRA_DETACH" ]; then $EXTRA_DETACH fi if [ -n "$EXTRA_UNMOUNT" ]; then $EXTRA_UNMOUNT fi } function trap_cleanup() { exitval=$? cleanup exit $exitval } function cleanup () { dib-block-device umount unmount_image cleanup_build_dir cleanup_image_dir } # Helper function to run a command inside the chroot function run_in_target () { cmd="$@" # -E to preserve http_proxy ORIG_HOME=$HOME export HOME=/root # Force an empty TMPDIR inside the chroot. There is no need to use an user # defined tmp dir which may not exist in the chroot. # Bug: #1330290 # Force the inclusion of a typical set of dirs in PATH, this is needed for guest # distros that have path elements not in the host PATH. sudo -E chroot $TMP_MOUNT_PATH env -u TMPDIR -u VIRTUAL_ENV PATH="\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" sh -c "$cmd" export HOME=$ORIG_HOME } # Helper function to run a directory of scripts inside the chroot function run_d_in_target () { check_element # If we can find a directory of hooks to run in the target filesystem, bind # mount it into the target and then execute run-parts in a chroot if [ -d ${TMP_HOOKS_PATH}/$1.d ] ; then sudo mkdir $TMP_MOUNT_PATH/tmp/in_target.d sudo mount --bind ${TMP_HOOKS_PATH} $TMP_MOUNT_PATH/tmp/in_target.d # Copy in dib-run-parts to run inside chroot. Note, in the # future, we might like to use a diffrent dib-run-parts for # running inside the chroot that doesn't rely on bash. For now # they're the same. Note also this gets cleaned up with the dir # delete below. sudo cp ${DIB_RUN_PARTS} ${TMP_MOUNT_PATH}/tmp/in_target.d sudo mount -o remount,ro,bind ${TMP_HOOKS_PATH} $TMP_MOUNT_PATH/tmp/in_target.d check_break before-$1 run_in_target bash [ -z "$break_outside_target" ] && in_target_arg="run_in_target" || in_target_arg= trap "check_break after-error $in_target_arg ${break_cmd:-bash}" ERR # NOTE: this is the dib-run-parts copied into the chroot by the # dib-run-parts element. run_in_target /tmp/in_target.d/dib-run-parts /tmp/in_target.d/$1.d trap - ERR check_break after-$1 run_in_target bash sudo umount -f $TMP_MOUNT_PATH/tmp/in_target.d if ! timeout 10 sh -c " while ! sudo rmdir $TMP_MOUNT_PATH/tmp/in_target.d; do sleep 1; done"; then echo "ERROR: unable to cleanly remove $TMP_MOUNT_PATH/tmp/in_target.d" exit 1 fi fi } function finalise_base () { TARGET_ROOT=$TMP_MOUNT_PATH run_d cleanup # In create_base() we replaced/created the initial resolv.conf # inside the image with a copy of the "outside" version so that # resolving during the build will work. # # If that file has been replace with a symlink (resolvconf package # can do this), or marked immutable, then don't restore the # original version, just leave it alone. if [ -L $TMP_MOUNT_PATH/etc/resolv.conf ] || \ lsattr $TMP_MOUNT_PATH/etc/resolv.conf | grep '^....i' >/dev/null ; then # We're keeping the contents of resolv.conf set in the elements, # so remove the old saved file sudo rm -f $TMP_MOUNT_PATH/etc/resolv.conf.ORIG else # Remove the resolv.conf we created above sudo rm -f $TMP_MOUNT_PATH/etc/resolv.conf # Move the original back if [ -L $TMP_MOUNT_PATH/etc/resolv.conf.ORIG ] || [ -f $TMP_MOUNT_PATH/etc/resolv.conf.ORIG ] ; then sudo mv $TMP_MOUNT_PATH/etc/resolv.conf.ORIG $TMP_MOUNT_PATH/etc/resolv.conf fi fi # Cleanup /tmp in the guest, so there is less cruft left there unmount_dir $TMP_MOUNT_PATH/tmp find $TMP_MOUNT_PATH/tmp -maxdepth 1 -mindepth 1 | xargs sudo rm -rf --one-file-system # Truncate /var/log files in preparation for first boot sudo find ${TMP_MOUNT_PATH}/var/log -type f -exec cp /dev/null '{}' \; # also /root logs sudo find ${TMP_MOUNT_PATH}/root -name \*.log -type f -delete } function compress_and_save_image () { # Recreate our image to throw away unnecessary data test $IMAGE_TYPE != qcow2 && COMPRESS_IMAGE="" if [ -n "$QEMU_IMG_OPTIONS" ]; then EXTRA_OPTIONS="-o $QEMU_IMG_OPTIONS" else EXTRA_OPTIONS="" fi if [ "$IMAGE_TYPE" = "raw" ]; then mv $TMP_IMAGE_PATH $1-new elif [ "$IMAGE_TYPE" == "tgz" ]; then gzip -9 < $IMAGE_NAME.tar > $1-new rm $IMAGE_NAME.tar elif [ "$IMAGE_TYPE" == "vhd" ]; then cp $TMP_IMAGE_PATH $1-intermediate vhd-util convert -s 0 -t 1 -i $1-intermediate -o $1-intermediate vhd-util convert -s 1 -t 2 -i $1-intermediate -o $1-new # The previous command creates a .bak file rm $1-intermediate.bak OUT_IMAGE_PATH=$1-new else echo "Converting image using qemu-img convert" qemu-img convert ${COMPRESS_IMAGE:+-c} -f raw -O $IMAGE_TYPE $EXTRA_OPTIONS $TMP_IMAGE_PATH $1-new fi OUT_IMAGE_PATH=$1-new finish_image $1 } function do_extra_package_install () { # Install any packages that were requested with the -p command line option if [ "$INSTALL_PACKAGES" != "" ]; then run_in_target install-packages ${INSTALL_PACKAGES[@]} fi } function copy_elements_lib () { sudo mkdir -p $TMP_MOUNT_PATH/lib/diskimage-builder sudo cp -t $TMP_MOUNT_PATH/lib/diskimage-builder $_LIB/elements-functions } # Dig up the initrd and kernel. function select_boot_kernel_initrd () { TARGET_ROOT=$1 BOOTDIR=$TARGET_ROOT/boot if [ -n "${DIB_BAREMETAL_KERNEL_PATTERN:-}" -a -n "${DIB_BAREMETAL_INITRD_PATTERN:-}" ]; then KERNEL=$(basename $(eval ls -1rv "$BOOTDIR/${DIB_BAREMETAL_KERNEL_PATTERN}" | head -1)) RAMDISK=$(basename $(eval ls -1rv "$BOOTDIR/${DIB_BAREMETAL_INITRD_PATTERN}" | head -1)) elif [ -f $TARGET_ROOT/etc/redhat-release ]; then # Prioritize PAE if present KERNEL=$(ls -1rv $BOOTDIR/vmlinuz* | grep PAE | grep -v debug | head -1 || echo "") KERNEL=${KERNEL:-$(ls -1rv $BOOTDIR/vmlinuz* | grep -v debug | head -1 || echo "")} if [ ! $KERNEL ]; then echo "No suitable kernel found." exit 1 fi KERNEL=$(basename $KERNEL) KERNEL_VERSION=${KERNEL#vmlinuz-} RAMDISK=$(basename $(ls $BOOTDIR/initramfs-$KERNEL_VERSION.img) || echo "") if [ ! $RAMDISK ]; then echo "Can't find an initramfs for the $KERNEL_VERSION version of the kernel." exit 1 fi elif [ -f $TARGET_ROOT/etc/debian_version ]; then KERNEL=$(basename $(ls -1rv $BOOTDIR/vmlinu*generic 2>/dev/null || ls -1rv $BOOTDIR/vmlinu* | head -1)) RAMDISK=$(basename $(ls -1rv $BOOTDIR/initrd*generic 2>/dev/null || ls -1rv $BOOTDIR/initrd* | head -1)) if [ -f $TARGET_ROOT/dib-signed-kernel-version ] ; then . $TARGET_ROOT/dib-signed-kernel-version fi if [ -n "${DIB_SIGNED_KERNEL_VERSION:-}" ]; then echo "Using signed kernel $DIB_SIGNED_KERNEL_VERSION" KERNEL=$(basename $(ls -1rv $BOOTDIR/vmlinu*generic.efi.signed 2>/dev/null)) fi elif [ -f $TARGET_ROOT/etc/SuSE-release ]; then KERNEL=$(basename $(readlink -e $BOOTDIR/vmlinuz)) RAMDISK=$(basename $(readlink -e $BOOTDIR/initrd)) elif [[ -f "${TARGET_ROOT}"/etc/gentoo-release ]]; then KERNEL="$(basename $(ls -1rv $BOOTDIR/kernel-*-openstack | head -n 1))" RAMDISK="$(basename $(ls -1rv $BOOTDIR/initramfs-*-openstack | head -n 1))" else echo "ERROR: Unable to detect operating system" exit 1 fi } diskimage-builder-2.11.0/diskimage_builder/lib/outfilter.py000066400000000000000000000056161324273057700237760ustar00rootroot00000000000000#!/usr/bin/env python # # Copyright 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # This is an output filter to filter and timestamp the logs from Grenade and # DevStack. Largely our awk filters got beyond the complexity level which were # sustainable, so this provides us much more control in a single place. # # The overhead of running python should be less than execing `date` a million # times during a run. import argparse import datetime import re import sys IGNORE_LINES = re.compile('(set \+o|xtrace)') def get_options(): parser = argparse.ArgumentParser( description='Filter output by DevStack and friends') parser.add_argument('-o', '--outfile', help='Output file for content', default=None) parser.add_argument('-v', '--verbose', action='store_true', help='Write to stdout', default=False) parser.add_argument('-b', '--no-timestamp', action='store_true', help='Do not prefix stdout with timestamp (bare)', default=False) return parser.parse_args() def skip_line(line): """Should we skip this line.""" return IGNORE_LINES.search(line) is not None def main(): opts = get_options() outfile = None if opts.outfile: outfile = open(opts.outfile, 'ab', 0) # Otherwise fileinput reprocess args as files sys.argv = [] for line in iter(sys.stdin.readline, ''): # put skip lines here if skip_line(line): continue now = datetime.datetime.utcnow() ts_line = ("%s | %s" % ( now.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3], line)) if opts.verbose: sys.stdout.write(line if opts.no_timestamp else ts_line) sys.stdout.flush() if outfile: # We've opened outfile as a binary file to get the # non-buffered behaviour. on python3, sys.stdin was # opened with the system encoding and made the line into # utf-8, so write the logfile out in utf-8 bytes. if sys.version_info < (3,): outfile.write(ts_line) else: outfile.write(ts_line.encode('utf-8')) outfile.flush() if __name__ == '__main__': try: sys.exit(main()) except KeyboardInterrupt: sys.exit(1) diskimage-builder-2.11.0/diskimage_builder/lib/ramdisk-defaults000066400000000000000000000016661324273057700245720ustar00rootroot00000000000000#!/bin/bash # Copyright 2012 Hewlett-Packard Development Company, L.P. # Copyright (c) 2012 NTT DOCOMO, INC. # # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # options for ramdisk-image-create export DIB_NO_TMPFS=${DIB_NO_TMPFS:-1} export DIB_MIN_TMPFS=${DIB_MIN_TMPFS:-1} export RAMDISK_ELEMENT=${RAMDISK_ELEMENT:-ramdisk} source $_LIB/common-defaults MODULE_ROOT=${DIB_MODULE_ROOT:-""} LIB_UDEV_ROOT=${DIB_LIB_UDEV_ROOT:-""} diskimage-builder-2.11.0/diskimage_builder/lib/ramdisk-functions000066400000000000000000000172431324273057700247710ustar00rootroot00000000000000#!/bin/bash # Copyright 2012 Hewlett-Packard Development Company, L.P. # Copyright (c) 2012 NTT DOCOMO, INC. # # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. function fullpath() { local f=$1 if [ "${f#/}" = "$f" ]; then echo `pwd`/"$f" else echo "$f" fi } function create_ramdisk_base () { echo "Creating base system" mkdir -p "${TMP_MOUNT_PATH}/"{bin,lib/modules,etc/udev} ln -s bin "$TMP_MOUNT_PATH/sbin" # cjk adding for discovery support mkdir -p "${TMP_MOUNT_PATH}/"{lib/udev/rules.d,var/{lib/dhcp,run}} case "$DISTRO_NAME" in fedora|rhel|rhel7|opensuse) mkdir -p "$TMP_MOUNT_PATH/usr" ln -s ../lib "$TMP_MOUNT_PATH/usr/lib" if [[ "`uname -m`" =~ x86_64|ppc64 ]]; then ln -s lib "$TMP_MOUNT_PATH/lib64" fi ;; esac if [ -e $LIB_UDEV/rules.d/50-firmware.rules ]; then cp -a "$LIB_UDEV/rules.d/50-firmware.rules" "$TMP_MOUNT_PATH/lib/udev/rules.d" fi cp -a "$LIB_UDEV/rules.d/80-drivers.rules" "$TMP_MOUNT_PATH/lib/udev/rules.d" if [ -a $LIB_UDEV/firmware ]; then cp -a "$LIB_UDEV/firmware" "$TMP_MOUNT_PATH/lib/udev" fi # cjk adding dhclient for hwdiscovery support # dhclient scripts on some distros appear in different places, copy any we find for FILE in /sbin/dhclient-script /usr/sbin/dhclient-script /etc/sysconfig/network-scripts/* /etc/rc.d/init.d/functions /etc/init.d/functions ; do if [ -f $FILE ] ; then mkdir -p $(dirname $TMP_MOUNT_PATH/$FILE) cp $FILE $TMP_MOUNT_PATH/$FILE fi done # /var/lib/dhclient is a directory on Fedora if [ -d "/var/lib/dhclient" ] ; then mkdir -p "$TMP_MOUNT_PATH/var/lib/dhclient" fi mkdir -p "$TMP_MOUNT_PATH/etc/modprobe.d" # The directory may or may not exist in the image. If the directory exists in # the image, all the files under it should get copied to the ramdisk. if [ -d "/etc/modprobe.d/" ] ; then find /etc/modprobe.d -name '*.conf' -type f -exec cp -a {} "$TMP_MOUNT_PATH/etc/modprobe.d" \; fi echo "blacklist evbug" > "$TMP_MOUNT_PATH/etc/modprobe.d/blacklist-dib-ramdisk.conf" # cjk adding for hwdiscovery support touch "$TMP_MOUNT_PATH/etc/fstab" mkdir -p "$TMP_MOUNT_PATH/etc/udev" cat >"$TMP_MOUNT_PATH/etc/udev/udev.conf" < /lib/x86_64-linux-gnu/libc.so.6 (0x00007f095e784000) # 2. only path # /lib64/ld-linux-x86-64.so.2 (0x00007f095ef79000) # 3. path to path # /lib64/ld-linux-x86-64.so.2 => /lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 (0x00007facff857000) # 4. name to empty (vdso) # linux-vdso.so.1 => (0x00007fff0c5ff000) # or, in some setups: # linux-vdso.so.1 (0x00007fff0c5ff000) for i in `echo "$ldd_out" | sed -e 's/^\t*//'`; do local ref=$( echo "$i" | awk -F '[ ]' '{print $1}') local real=$( echo "$i" | awk -F '[ ]' '$2 == "=>" {print $3} $2 != "=>" {print $1}') if [ -z "$real" ] || [[ "$real" != /* ]]; then continue fi if [ "$ref" = "${ref#/}" ]; then ref=/lib/$ref fi dest=/lib/`basename "$real"` cp -Ln "$real" "$TMP_MOUNT_PATH/$dest" # Create a symbolic link if the shared library is referred # by the different name if [ "$ref" != "$dest" ]; then local link_path=$TMP_MOUNT_PATH/$ref if ! [ -e "$link_path" -o -L "$link_path" ]; then mkdir -p $(dirname "$link_path") ln -s "$dest" "$link_path" fi fi done } function populate_lib () { echo "Populating /lib" # find udevd UDEVD= for f in /sbin/udevd /lib/udev/udevd /lib/systemd/systemd-udevd \ /usr/lib/systemd/systemd-udevd \ /usr/lib/udev/udevd; do if [ -x "$f" ]; then UDEVD="$f" break fi done UDEV_FIRMWARE= if [ -a $LIB_UDEV/firmware ]; then UDEV_FIRMWARE="$LIB_UDEV/firmware" fi for i in "$BUSYBOX" bash lsmod modprobe udevadm \ wget reboot shutdown $UDEVD $UDEV_FIRMWARE \ $(cat /etc/dib_binary_deps) ; do # Don't take the ip command from busybox, its missing some features if busybox_list | grep -v "^ip$" | grep "^$i\$" >/dev/null; then continue fi path=`type -p $i 2>/dev/null` || path=$i if ! [ -x "$path" ]; then echo "$i is not found in PATH" 2>&1 exit 1 fi cp -L "$path" "$TMP_MOUNT_PATH/bin/" copy_required_libs "$path" done if [ -f /dib-signed-kernel-version ] ; then . /dib-signed-kernel-version fi if [ -n "${DIB_SIGNED_KERNEL_VERSION:-}" ]; then # Secure kernel module directory does not have efi.signed suffix to # kernel version. if echo $KERNEL_VERSION | grep -q 'efi.signed'; then KERNEL_VERSION=`echo "$KERNEL_VERSION" |sed "s/\.efi\.signed//g"` fi fi cp -a "$MODULE_DIR" "$TMP_MOUNT_PATH/lib/modules/$KERNEL_VERSION" echo "Removing kernel framebuffer drivers to enforce text mode consoles..." find $TMP_MOUNT_PATH/lib/modules/$KERNEL_VERSION/kernel/drivers/video -name '*fb.ko' -exec rm -v {} + if [ -d $FIRMWARE_DIR ]; then cp -a "$FIRMWARE_DIR" "$TMP_MOUNT_PATH/lib/firmware" fi } function busybox_list () { # busybox supports --list option since version 1.18 "$BUSYBOX" --list 2> /dev/null && return # for busybox under 1.18 we parse command list from --help output scrlet='{ if (go) { print } } /Currently defined functions:/ { go=1 }' "$BUSYBOX" --help | awk "$scrlet" | tr ',' '\n' | xargs -n1 echo } function populate_busybox () { echo "Creating symlinks for busybox binaries" for i in $( busybox_list ); do if [ -f "$TMP_MOUNT_PATH/bin/$i" ]; then echo "skip $i" continue fi ln -s busybox "$TMP_MOUNT_PATH/bin/$i" done } function populate_init () { echo "Installing init" cp "$INIT" "$TMP_MOUNT_PATH/init" chmod +x $TMP_MOUNT_PATH/init for F in "$FUNCTIONS_D"/* ; do cp "$F" "$TMP_MOUNT_PATH" done # Append /init with any element fragments that are present TARGET_DIR="/tmp/in_target.d/" for _ELEMENT in $(ls $TARGET_DIR/init.d/) ; do _FILE="${TARGET_DIR}/init.d/${_ELEMENT}" if [ -a $_FILE ]; then cat >>$TMP_MOUNT_PATH/init <>$TMP_MOUNT_PATH/init fi done # Add our final steps to /init cat <${INIT}-end >>$TMP_MOUNT_PATH/init } function finalise_image () { echo "Finalising image" (cd "$TMP_MOUNT_PATH"; find . | cpio -o -H newc | gzip > "$TMP_IMAGE_PATH" ) } function populate_udev () { echo "Installing udev rules" TARGET_DIR="/tmp/in_target.d/" for _ELEMENT in $(ls $TARGET_DIR/udev.d/) ; do _FILE="${TARGET_DIR}/udev.d/${_ELEMENT}" if [ -a $_FILE ]; then cp ${_FILE} $TMP_MOUNT_PATH/lib/udev/rules.d/ fi done } function find_kernel_version () { _TMP=$(ls /boot/vmlinu* | sort | tail -1) if [ "$_TMP" == "" ]; then echo "Unable to find a suitable kernel" >>/dev/stderr exit 1 fi echo ${_TMP##/boot/vmlinu[zx]-} } diskimage-builder-2.11.0/diskimage_builder/lib/ramdisk-image-create000077700000000000000000000000001324273057700304662disk-image-createustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/logging_config.py000066400000000000000000000062251324273057700241630ustar00rootroot00000000000000# Copyright 2016 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # Python Logging Configuration for DIB # Usage: # In the main (application) file, do an # import logging_config # ... # logging_config.setup() # It is then possible to use the normal python logging interface, like # logger = logging.getLogger(__name__) # logger.info("Info Message") import json import logging import logging.config import os import sys # A simple formatter to more or less copy oslo.log's ContextFormatter class DibFormatter(logging.Formatter): def __init__(self, *args, **kwargs): self.fmt = kwargs['fmt'] self.debug_suffix = kwargs.pop('debug_suffix') logging.Formatter.__init__(self, *args, **kwargs) def format(self, record): if record.levelno == logging.DEBUG: fmt = self.fmt + self.debug_suffix else: fmt = self.fmt if sys.version_info < (3, 2): self._fmt = fmt else: self._style = logging.PercentStyle(fmt) self._fmt = self._style._fmt return logging.Formatter.format(self, record) def setup(): # Check for the DIB_DEBUG_TRACE environment variable # If it is set to something greater then 0, use debug # logging. LOG_LEVEL = "DEBUG" if 'DIB_DEBUG_TRACE' in os.environ \ and int(os.environ['DIB_DEBUG_TRACE']) > 0 else "INFO" # Default logging configuration which can be overwritten # by a config file passed in by a user. PYTHON_LOGGING_CONFIG = { 'version': 1, 'disable_existing_loggers': False, # note that disk-image-create runs stdout through # outfilter.py, which adds the timestamp. this doesn't have a # timestamp to avoid double logging 'formatters': { 'standard': { '()': 'diskimage_builder.logging_config.DibFormatter', 'fmt': '%(levelname)s ' '%(name)s [-] %(message)s', 'debug_suffix': ' %(funcName)s %(pathname)s:%(lineno)d' } }, 'handlers': { 'default': { 'level': LOG_LEVEL, 'class': 'logging.StreamHandler', 'formatter': 'standard', } }, 'loggers': { '': { 'handlers': ['default'], 'level': LOG_LEVEL, 'propagate': True } } } if 'DIB_PYTHON_LOGGING_CONFIG_FILE' in os.environ: with open(os.environ['DIB_PYTHON_LOGGING_CONFIG_FILE']) as fd: PYTHON_LOGGING_CONFIG = json.load(fd) logging.config.dictConfig(PYTHON_LOGGING_CONFIG) diskimage-builder-2.11.0/diskimage_builder/paths.py000066400000000000000000000020171324273057700223220ustar00rootroot00000000000000# Copyright 2016 Ian Wienand (iwienand@redhat.com) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. """Export paths""" import os import pkg_resources import sys def get_path(var): if var == "lib": return os.path.abspath( pkg_resources.resource_filename(__name__, "lib")) elif var == "elements": return os.path.abspath( pkg_resources.resource_filename(__name__, "elements")) else: print("Unknown path request!") sys.exit(1) def show_path(var): print(get_path(var)) diskimage-builder-2.11.0/diskimage_builder/tests/000077500000000000000000000000001324273057700217735ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/tests/__init__.py000066400000000000000000000000001324273057700240720ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/tests/base.py000066400000000000000000000040271324273057700232620ustar00rootroot00000000000000# Copyright 2014 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # TODO(bnemec): This whole file is a copy-paste of the one in # tripleo-image-elements. That duplication needs to be eliminated. import os import subprocess import sys import tempfile from oslotest import base class ScriptTestBase(base.BaseTestCase): def setUp(self): super(ScriptTestBase, self).setUp() self.tmpdir = tempfile.mkdtemp() self.env = os.environ.copy() self.env['PATH'] = self.tmpdir + ':' + self.env['PATH'] def _stub_script(self, name, contents): filename = os.path.join(self.tmpdir, name) with open(filename, 'w') as f: f.write('#!/bin/bash\n') f.write(contents) f.write('\n') os.chmod(filename, 0o700) def _run_command(self, cmd): try: # check_output doesn't exist in Python < 2.7 if sys.hexversion < 0x02070000: runner = subprocess.check_call else: runner = subprocess.check_output return runner(cmd, stderr=subprocess.STDOUT, env=self.env) # NOTE(bnemec): Handle the exception so we can extract as much # information as possible. except subprocess.CalledProcessError as e: # The check_call exception won't have any data in e.output if sys.hexversion < 0x02070000: self.fail(e) else: self.fail(e.output) diskimage-builder-2.11.0/diskimage_builder/tests/functional/000077500000000000000000000000001324273057700241355ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/tests/functional/__init__.py000066400000000000000000000000001324273057700262340ustar00rootroot00000000000000diskimage-builder-2.11.0/diskimage_builder/tests/test_elementdeps.py000066400000000000000000000222671324273057700257220ustar00rootroot00000000000000# Copyright 2013 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import logging import os import fixtures import testtools from diskimage_builder import element_dependencies logger = logging.getLogger(__name__) data_dir = os.path.abspath( os.path.join(os.path.dirname(__file__), 'test-elements')) def _populate_element(element_dir, element_name, element_deps=[], provides=[]): logger.debug("Populate %s <%s>", element_name, element_dir) element_home = os.path.join(element_dir, element_name) os.mkdir(element_home) deps_path = os.path.join(element_home, 'element-deps') with open(deps_path, 'w') as deps_file: deps_file.write("\n".join(element_deps)) provides_path = os.path.join(element_home, 'element-provides') with open(provides_path, 'w') as provides_file: provides_file.write("\n".join(provides)) class TestElementDeps(testtools.TestCase): def setUp(self): super(TestElementDeps, self).setUp() self.element_root_dir = self.useFixture(fixtures.TempDir()).path self.element_dir = os.path.join(self.element_root_dir, 'elements') self.element_override_dir = os.path.join(self.element_root_dir, 'element-override') os.mkdir(self.element_dir) os.mkdir(self.element_override_dir) self.log_fixture = self.useFixture( fixtures.FakeLogger(level=logging.DEBUG)) _populate_element(self.element_dir, 'requires-foo', ['foo']) _populate_element(self.element_dir, 'foo', [], ['operating-system']) _populate_element(self.element_dir, 'requires-requires-foo', ['requires-foo']) _populate_element(self.element_dir, 'self', ['self']) _populate_element(self.element_dir, 'provides_virtual', [], ['virtual']) _populate_element(self.element_dir, 'requires_virtual', ['virtual'], ['operating-system']) _populate_element(self.element_dir, 'virtual', ['extra_dependency']) _populate_element(self.element_dir, 'extra_dependency', []) _populate_element(self.element_dir, 'circular1', ['circular2'], ['operating-system']) _populate_element(self.element_dir, 'circular2', ['circular1']) _populate_element(self.element_dir, 'provides_new_virtual', [], ['new_virtual', 'operating-system']) _populate_element(self.element_dir, 'requires_new_virtual', ['new_virtual']) # second element should override the first one here _populate_element(self.element_dir, 'override_element', []) _populate_element(self.element_override_dir, 'override_element', []) # This simulates $ELEMENTS_PATH self.element_dirs = "%s:%s" % (self.element_override_dir, self.element_dir) # helper to return an (element, path) tuple from the standard dir def _e(self, element): return (element, os.path.join(self.element_dir, element)) # helper to return an (element, path) tuple from the override dir def _eo(self, element): return (element, os.path.join(self.element_override_dir, element)) def test_non_transitive_deps(self): result = element_dependencies.get_elements(['requires-foo'], self.element_dirs) self.assertItemsEqual([self._e('foo'), self._e('requires-foo')], result) def test_missing_deps(self): e = self.assertRaises(element_dependencies.MissingElementException, element_dependencies.get_elements, ['fake'], self.element_dirs) self.assertIn("Element 'fake' not found", str(e)) def test_invalid_element_dir(self): e = self.assertRaises(element_dependencies.InvalidElementDir, element_dependencies.get_elements, ['fake'], self.element_dirs + ":/not/a/dir") self.assertIn("ELEMENTS_PATH entry '/not/a/dir' is not a directory", str(e)) def test_transitive_deps(self): result = element_dependencies.get_elements( ['requires-requires-foo'], self.element_dirs) self.assertItemsEqual([self._e('requires-requires-foo'), self._e('requires-foo'), self._e('foo')], result) def test_no_deps(self): result = element_dependencies.get_elements(['foo'], self.element_dirs) self.assertEqual([self._e('foo')], result) def test_self(self): result = element_dependencies.get_elements(['self', 'foo'], self.element_dirs) self.assertItemsEqual([self._e('self'), self._e('foo')], result) def test_circular(self): result = element_dependencies.get_elements(['circular1'], self.element_dirs) self.assertItemsEqual([self._e('circular1'), self._e('circular2')], result) def test_provide(self): result = element_dependencies.get_elements( ['provides_virtual', 'requires_virtual'], self.element_dirs) self.assertItemsEqual([self._e('requires_virtual'), self._e('provides_virtual')], result) def test_provide_conflict(self): self.assertRaises(element_dependencies.AlreadyProvidedException, element_dependencies.get_elements, ['virtual', 'provides_virtual'], self.element_dirs) def test_provide_virtual_ordering(self): result = element_dependencies.get_elements( ['requires_new_virtual', 'provides_new_virtual'], self.element_dirs) self.assertItemsEqual( [self._e('requires_new_virtual'), self._e('provides_new_virtual')], result) def test_no_os_element(self): self.assertRaises(element_dependencies.MissingOSException, element_dependencies.get_elements, ['provides_virtual'], self.element_dirs) def test_duplicated_os_passed_as_element(self): self.assertRaises( element_dependencies.AlreadyProvidedException, element_dependencies.get_elements, ['circular1', 'operating-system'], self.element_dirs) # ensure we get the error message about what's providing the # conflicting package self.assertIn("operating-system : already provided by ['circular1']", self.log_fixture.output) def test_element_override(self): # make sure we picked up "override_element" from the override dir, # not the base dir result = element_dependencies.get_elements(['override_element', 'foo'], self.element_dirs) self.assertItemsEqual([self._e('foo'), self._eo('override_element')], result) def test_expand_dependencies_deprecated(self): # test the deprecated expand_dependencies call result = element_dependencies.expand_dependencies( ['foo', 'requires-foo'], self.element_dirs) self.assertItemsEqual(['foo', 'requires-foo'], result) def test_output_sanity(self): # very basic output sanity test elements = element_dependencies._get_elements(['foo', 'requires-foo'], self.element_dirs) element_dependencies._output_env_vars(elements) class TestElements(testtools.TestCase): def test_depends_on_env(self): self.useFixture( fixtures.EnvironmentVariable('ELEMENTS_PATH', '/foo/bar')) self.assertEqual('/foo/bar', element_dependencies._get_elements_dir()) def test_env_not_set(self): self.useFixture(fixtures.EnvironmentVariable('ELEMENTS_PATH', '')) self.assertRaises(Exception, element_dependencies._get_elements_dir, ()) diskimage-builder-2.11.0/diskimage_builder/tests/test_loggingconfig.py000066400000000000000000000037231324273057700262250ustar00rootroot00000000000000# Copyright 2016 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import fixtures import logging import testtools from diskimage_builder import logging_config class TestLoggingConfig(testtools.TestCase): def test_defaults(self): self.out = self.useFixture(fixtures.StringStream('stdout')) self.useFixture( fixtures.MonkeyPatch('sys.stdout', self.out.stream)) self.err = self.useFixture(fixtures.StringStream('stderr')) self.useFixture( fixtures.MonkeyPatch('sys.stderr', self.err.stream)) self.useFixture(fixtures.EnvironmentVariable('DIB_DEBUG_TRACE', '1')) logging_config.setup() log = logging.getLogger(__name__) log.debug("Debug Message") self.assertIn("Debug Message", self.err._details["stderr"].as_text()) # The follow two are looking for the function name / file name # suffix we log only for debug messages self.assertIn("test_defaults", self.err._details["stderr"].as_text()) self.assertIn("test_loggingconfig.py", self.err._details["stderr"].as_text()) log.info("Info Message") self.assertIn("Info Message", self.err._details["stderr"].as_text()) log.warning("Warning Message") self.assertIn("Warning Message", self.err._details["stderr"].as_text()) log.error("Error Message") self.assertIn("Error Message", self.err._details["stderr"].as_text()) diskimage-builder-2.11.0/diskimage_builder/tests/test_no_dup_filenames.py000066400000000000000000000031431324273057700267140ustar00rootroot00000000000000# Copyright 2014 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import glob import os import testtools class TestNoDupFilenames(testtools.TestCase): def test_no_dup_filenames(self): topdir = os.path.normpath(os.path.dirname(__file__) + '/../../') elements_glob = os.path.join(topdir, "elements", "*") filenames = [] dirs_to_check = ['block-device.d', 'cleanup.d', 'extra-data.d', 'finalise.d', 'install.d', 'post-install.d', 'pre-install.d', 'root.d'] for element_dir in glob.iglob(elements_glob): for dir_to_check in dirs_to_check: target_dir = os.path.join(element_dir, dir_to_check, "*") for target in glob.iglob(target_dir): short_path = target[len(element_dir) + 1:] if not os.path.isdir(target): err_msg = 'Duplicate file name found %s' % short_path self.assertFalse(short_path in filenames, err_msg) filenames.append(short_path) diskimage-builder-2.11.0/diskimage_builder/version.py000066400000000000000000000013141324273057700226670ustar00rootroot00000000000000# Copyright 2011 OpenStack Foundation # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import pbr.version version_info = pbr.version.VersionInfo('diskimage-builder') diskimage-builder-2.11.0/doc/000077500000000000000000000000001324273057700157335ustar00rootroot00000000000000diskimage-builder-2.11.0/doc/lib/000077500000000000000000000000001324273057700165015ustar00rootroot00000000000000diskimage-builder-2.11.0/doc/lib/element_deps.py000066400000000000000000000044451324273057700215260ustar00rootroot00000000000000# # Copyright 2017 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # # Scan the element directory, looks for element dependencies and # writes them out when the directive # .. element_deps:: # is used. # This was developed only for internal use and must be called # from the source top directory. from diskimage_builder.element_dependencies import _find_all_elements from diskimage_builder.paths import get_path from docutils.parsers.rst import Directive import os all_elements = _find_all_elements(get_path("elements")) def make_dep_list(title, deps): lines = [] lines.append(title) lines.append("+" * len(title)) for dep in deps: lines.append("* :doc:`../%s/README`" % dep) lines.append('') # careful to end with a blank line return lines class ElementDepsDirective(Directive): # this enables content in the directive has_content = True def run(self): source = self.state_machine.input_lines.source( self.lineno - self.state_machine.input_offset - 1) # Extract the element from the source attribute of the document element_name = os.path.basename(os.path.dirname( self.state_machine.document.attributes['source'])) lines = ["Element Dependencies", "--------------------"] # This should not fail -- sphinx would be finding an element # that dib doesn't know about? element = all_elements[element_name] if element.depends: lines.extend(make_dep_list("Uses", element.depends)) if element.r_depends: lines.extend(make_dep_list("Used by", element.r_depends)) self.state_machine.insert_input(lines, source) return [] def setup(app): app.add_directive('element_deps', ElementDepsDirective) diskimage-builder-2.11.0/doc/source/000077500000000000000000000000001324273057700172335ustar00rootroot00000000000000diskimage-builder-2.11.0/doc/source/ci.md000066400000000000000000000025021324273057700201470ustar00rootroot00000000000000CI needs for image building =========================== Eventually, if/when TripleO becomes an official OpenStack project, all CI for it should be on OpenStack systems. Until then we still need CI. Jenkins ------- * Jenkins from jenkins apt repo. * IRC notification service, notify-only on #triple on freenode, port 7000 ssl. * Github OAuth plugin, permit all from tripleo organisation, and organisation members as service admins. * Grant jenkins builders sudo [may want lxc containers or cloud instances for security isolation] * Jobs to build: * base ubuntu VM. disk-image-create vm base -o base -a i386 * ramdisk deploy image build ramdisk-image-create deploy Copyright ========= Copyright 2012, 2013 Hewlett-Packard Development Company, L.P. Copyright (c) 2012 NTT DOCOMO, INC. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diskimage-builder-2.11.0/doc/source/conf.py000066400000000000000000000173571324273057700205470ustar00rootroot00000000000000import sys import os # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. sys.path.append(os.path.abspath('../lib')) extensions = ['openstackdocstheme', 'element_deps', 'sphinx.ext.autodoc'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'diskimage-builder' copyright = u'2015-2016, Various Authors' # openstackdocstheme options repository_name = 'openstack/diskimage-builder' bug_project = 'diskimage-builder' bug_tag = '' html_last_updated_fmt = '%Y-%m-%d %H:%M' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # import pbr.version version_info = pbr.version.VersionInfo('diskimage-builder') # The short X.Y version. release = version_info.release_string() # The full version, including alpha/beta/rc tags. version = version_info.version_string() # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build', 'doc/build', '.tox', '.venv', 'elements/*/test-elements'] # The reST default role (used for this markup: `text`) to use for all # documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. #keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'openstackdocs' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". #html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. #html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. html_sidebars = { '**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html'], } # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'diskimage-builderdoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ ('index', 'diskimage-builder.tex', u'diskimage-builder Documentation', u'OpenStack', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'diskimage-builder', u'diskimage-builder Documentation', [u'OpenStack'], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'diskimage-builder', u'diskimage-builder Documentation', u'OpenStack', 'diskimage-builder', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False diskimage-builder-2.11.0/doc/source/developer/000077500000000000000000000000001324273057700212205ustar00rootroot00000000000000diskimage-builder-2.11.0/doc/source/developer/caches.rst000066400000000000000000000027731324273057700232110ustar00rootroot00000000000000Caches and offline mode ======================= Since retrieving and transforming operating system image files, git repositories, Python or Ruby packages, and so on can be a significant overhead, we cache many of the inputs to the build process. The cache location is read from ``DIB_IMAGE_CACHE``. :ref:`developing-elements` describes the interface within disk-image-builder for caching. When invoking disk-image-builder, the ``--offline`` option will instruct disk-image-builder to not refresh cached resources. Alternatively you can set ``DIB_OFFLINE=1``. Note that we don't maintain operating system package caches, instead depending on your local infrastructure (e.g. Squid cache, or an APT or Yum proxy) to facilitate caching of that layer, so you need to arrange independently for offline mode. For more information about setting up a squid proxy, consult the `TripleO documentation `_. Base images ----------- These are cached by the standard elements - :doc:`../elements/fedora/README`, :doc:`../elements/redhat-common/README`, :doc:`../elements/ubuntu/README`, :doc:`../elements/debian/README` and :doc:`../elements/opensuse/README`. source-repositories ------------------- Git repositories and tarballs obtained via the :doc:`../elements/source-repositories/README` element will be cached. PyPI ---- The :doc:`../elements/pypi/README` element will bind mount a PyPI mirror from the cache dir and configure pip and easy-install to use it. diskimage-builder-2.11.0/doc/source/developer/components.rst000066400000000000000000000026221324273057700241410ustar00rootroot00000000000000Components ========== `disk-image-create [-a i386|amd64|armhf|arm64] -o filename {element} [{element} ...]` Create an image of element {element}, optionally mixing in other elements. Element dependencies are automatically included. Support for other architectures depends on your environment being able to run binaries of that platform and/or packages being available for the architecture. For instance, to enable armhf on Ubuntu install the qemu-user-static package, or to enable arm64 on CentOS setup the RDO aarch64 package repositories. The default output format from disk-image-create is qcow2. To instead output a tarball pass in "-t tar". This tarball could then be used as an image for a linux container(see docs/docker.md). `ramdisk-image-create -o filename {element} [{element} ...]` Create a kernel+ ramdisk pair for running maintenance on bare metal machines (deployment, inventory, burnin etc). To generate kernel+ramdisk pair for use with nova-baremetal, use:: ramdisk-image-create -o deploy.ramdisk deploy-baremetal To generate kernel+ramdisk pair for use with ironic, use:: ramdisk-image-create -o deploy.ramdisk ironic-agent `element-info` Extract information about elements. `tests/run_functests.sh` This runs a set of functional tests for diskimage-builder. elements can be found in the top level elements directory. diskimage-builder-2.11.0/doc/source/developer/design.rst000066400000000000000000000065451324273057700232350ustar00rootroot00000000000000Design ====== Images are built using a chroot and bind mounted /proc /sys and /dev. The goal of the image building process is to produce blank slate machines that have all the necessary bits to fulfill a specific purpose in the running of an OpenStack cloud: e.g. a nova-compute node. Images produce either a filesystem image with a label of cloudimg-rootfs, or can be customised to produce whole disk images (but will still contain a filesystem labelled cloudimg-rootfs). Once the file system tree is assembled a loopback device with filesystem (or partition table and file system) is created and the tree copied into it. The file system created is an ext4 filesystem just large enough to hold the file system tree and can be resized up to 1PB in size. To produce the smallest image the utility fstrim is used. When deleting a file the space is simply marked as free on the disk, the file is still there until it is overwritten. fstrim informs the underlying disk to drop those bytes the end result of which is like writting zeros over those sectors. The same effect could be achieved by creating a large file full of zeros and removing that file, however that method is far more IO intensive. An element is a particular set of code that alters how the image is built, or runs within the chroot to prepare the image. E.g. the local-config element copies in the http proxy and ssh keys of the user running the image build process into the image, whereas the vm element makes the image build a regular VM image with partition table and installed grub boot sector. The mellanox element adds support for mellanox infiniband hardware to both the deploy ramdisk and the built images. Images must specify a base distribution image element. Currently base distribution elements exist for fedora, rhel, ubuntu, debian and opensuse. Other distributions may be added in future, the infrastructure deliberately makes few assumptions about the exact operating system in use. The base image has opensshd running (a new key generated on first boot) and accepts keys via the cloud metadata service, loading them into the distribution specific default user account. The goal of a built image is to have any global configuration ready to roll, but nothing that ties it to a specific cloud instance: images should be able to be dropped into a test cloud and validated, and then deployed into a production cloud (usually via bare metal nova) for production use. As such, the image contents can be modelled as three distinct portions: - global content: the actual code, kernel, always-applicable config (like disabling password authentication to sshd). - metadata / config management provided configuration: user ssh keys, network address and routes, configuration management server location and public key, credentials to access other servers in the cloud. These are typically refreshed on every boot. - persistent state: sshd server key, database contents, swift storage areas, nova instance disk images, disk image cache. These would typically be stored on a dedicated partition and not overwritten when re-deploying the image. The goal of the image building tools is to create machine images that contain the correct global content and are ready for 'last-mile' configuration by the nova metadata API, after which a configuration management system can take over (until the next deploy, when it all starts over from scratch). diskimage-builder-2.11.0/doc/source/developer/developing_elements.rst000066400000000000000000000417171324273057700260140ustar00rootroot00000000000000.. _developing-elements: Developing Elements =================== Conform to the following conventions: * Use the environment for overridable defaults, prefixing environment variable names with ``DIB_``. For example: .. sourcecode:: sh DIB_MYDEFAULT=${DIB_MYDEFAULT:-default} If you do not use the ``DIB`` prefix you may find that your overrides are discarded as the build environment is sanitised. * Consider that your element co-exists with many others and try to guard against undefined behaviours. Some examples: * Two elements use the source-repositories element, but use the same filename for the source-repositories config file. Files such as these (and indeed the scripts in the various .d directories :ref:`listed below `) should be named such that they are unique. If they are not unique, when the combined tree is created by disk-image-builder for injecting into the build environment, one of the files will be overwritten. * Two elements copy different scripts into ``/usr/local/bin`` with the same name. If they both use ``set -e`` and ``cp -n`` then the conflict will be caught and cause the build to fail. * If your element mounts anything into the image build tree (``$TMP_BUILD_DIR``) then it will be automatically unmounted when the build tree is unmounted - and not remounted into the filesystem image - if the mount point is needed again, your element will need to remount it at that point. * If caching is required, elements should use a location under ``$DIB_IMAGE_CACHE``. * Elements should allow for remote data to be cached. When ``$DIB_OFFLINE`` is set, this cached data should be used if possible. See the :ref:`dev-global-image-build-variables` section of this document for more information. * Elements in the upstream diskimage-builder elements should not create executables which run before 10- or after 90- in any of the phases if possible. This is to give downstream elements the ability to easily make executables which run after our upstream ones. .. _phase-subdirectories: Phase Subdirectories ^^^^^^^^^^^^^^^^^^^^ Make as many of the following subdirectories as you need, depending on what part of the process you need to customise. The subdirectories are executed in the order given here. Scripts within the subdirectories should be named with a two-digit numeric prefix, and are executed in numeric order. Only files which are marked executable (+x) will be run, so other files can be stored in these directories if needed. As a convention, we try to only store executable scripts in the phase subdirectories and store data files elsewhere in the element. The phases are: #. ``root.d`` #. ``extra-data.d`` #. ``pre-install.d`` #. ``install.d`` #. ``post-install.d`` #. ``block-device.d`` #. ``finalise.d`` #. ``cleanup.d`` ``root.d`` Create or adapt the initial root filesystem content. This is where alternative distribution support is added, or customisations such as building on an existing image. Only one element can use this at a time unless particular care is taken not to blindly overwrite but instead to adapt the context extracted by other elements. * runs: **outside chroot** * inputs: * ``$ARCH=i386|amd64|armhf|arm64`` * ``$TARGET_ROOT=/path/to/target/workarea`` ``extra-data.d`` Pull in extra data from the host environment that hooks may need during image creation. This should copy any data (such as SSH keys, http proxy settings and the like) somewhere under ``$TMP_HOOKS_PATH``. * runs: **outside chroot** * inputs: ``$TMP_HOOKS_PATH`` * outputs: None ``pre-install.d`` Run code in the chroot before customisation or packages are installed. A good place to add apt repositories. * runs: **in chroot** ``install.d`` Runs after ``pre-install.d`` in the chroot. This is a good place to install packages, chain into configuration management tools or do other image specific operations. * runs: **in chroot** ``post-install.d`` Run code in the chroot. This is a good place to perform tasks you want to handle after the OS/application install but before the first boot of the image. Some examples of use would be: Run ``chkconfig`` to disable unneeded services Clean the cache left by the package manager to reduce the size of the image. * runs: **in chroot** ``block-device.d`` Customise the block device that the image will be made on (for example to make partitions). Runs after the target tree has been fully populated but before the ``cleanup.d`` phase runs. * runs: **outside chroot** * inputs: * ``$IMAGE_BLOCK_DEVICE={path}`` * ``$TARGET_ROOT={path}`` * outputs: ``$IMAGE_BLOCK_DEVICE={path}`` ``finalise.d`` Perform final tuning of the root filesystem. Runs in a chroot after the root filesystem content has been copied into the mounted filesystem: this is an appropriate place to reset SELinux metadata, install grub bootloaders and so on. Because this happens inside the final image, it is important to limit operations here to only those necessary to affect the filesystem metadata and image itself. For most operations, ``post-install.d`` is preferred. * runs: **in chroot** ``cleanup.d`` Perform cleanup of the root filesystem content. For instance, temporary settings to use the image build environment HTTP proxy are removed here in the dpkg element. * runs: outside chroot * inputs: * ``$ARCH=i386|amd64|armhf|arm64`` * ``$TARGET_ROOT=/path/to/target/workarea`` Other Subdirectories ^^^^^^^^^^^^^^^^^^^^ Elements may have other subdirectories that are processed by specific elements rather than the diskimage-builder tools themselves. One example of this is the ``bin`` directory. The `rpm-distro`, :doc:`../elements/dpkg/README` and :doc:`../elements/opensuse/README` elements install all files found in the ``bin`` directory into ``/usr/local/bin`` within the image as executable files. Environment Variables ^^^^^^^^^^^^^^^^^^^^^ To set environment variables for other hooks, add a file to your element ``environment.d``. This directory contains bash script snippets that are sourced before running scripts in each phase. Note that because environment includes are sourced together, they should not set global flags like ``set -x`` because they will affect all preceeding imports. Dependencies ^^^^^^^^^^^^ Each element can use the following files to define or affect dependencies: ``element-deps`` A plain text, newline separated list of elements which will be added to the list of elements built into the image at image creation time. ``element-provides`` A plain text, newline separated list of elements which are provided by this element. These elements will be excluded from elements built into the image at image creation time. For example if element A depends on element B and element C includes element B in its ``element-provides`` file and A and C are included when building an image, then B is not used. Operating system elements ^^^^^^^^^^^^^^^^^^^^^^^^^ Some elements define the base structure for an operating system -- for example, the ``opensuse`` element builds a base openSUSE system. Such elements have more requirements than the other elements: * they must have ``operating-system`` in their element-provides, so this indicates they are an "operating system". * they must export the ``DISTRO_NAME`` environment variable with the name of the distribution built, using an environment.d script. For example, the ``opensuse`` element exports ``DISTRO_NAME=opensuse``. Ramdisk Elements ^^^^^^^^^^^^^^^^ Ramdisk elements support the following files in their element directories: ``binary-deps.d`` Text files listing executables required to be fed into the ramdisk. These need to be present in ``$PATH`` in the build chroot (i.e. need to be installed by your elements as described above). ``init.d`` POSIX shell script fragments that will be appended to the default script executed as the ramdisk is booted (``/init``). ``ramdisk-install.d`` Called to copy files into the ramdisk. The variable ``$TMP_MOUNT_PATH`` points to the root of the tree that will be packed into the ramdisk. ``udev.d`` ``udev`` rules files that will be copied into the ramdisk. Element coding standard ^^^^^^^^^^^^^^^^^^^^^^^ - lines should not include trailing whitespace. - there should be no hard tabs in the file. - indents are 4 spaces, and all indentation should be some multiple of them. - `do` and `then` keywords should be on the same line as the if, while or for conditions. .. _dev-global-image-build-variables: Global image-build variables ---------------------------- ``DIB_OFFLINE`` This is always set. When not empty, any operations that perform remote data access should avoid it if possible. If not possible the operation should still be attempted as the user may have an external cache able to keep the operation functional. ``DIB_IMAGE_ROOT_FS_UUID`` This contains the UUID of the root filesystem, when diskimage-builder is building a disk image. This works only for ext filesystems. ``DIB_IMAGE_CACHE`` Path to where cached inputs to the build process are stored. Defaults to ``~/.cache/image_create``. Structure of an element ----------------------- The above-mentioned global content can be further broken down in a way that encourages composition of elements and reusability of their components. One possible approach to this would be to label elements as either a "driver", "service", or "config" element. Below are some examples. - Driver-specific elements should only contain the necessary bits for that driver:: elements/ driver-mellanox/ init - modprobe line install.d/ 10-mlx - package installation - An element that installs and configures Nova might be a bit more complex, containing several scripts across several phases:: elements/ service-nova/ source-repository-nova - register a source repository pre-install.d/ 50-my-ppa - add a PPA install.d/ 10-user - common Nova user accts 50-my-pack - install packages from my PPA 60-nova - install nova and some dependencies - In the general case, configuration should probably be handled either by the meta-data service (eg, o-r-c) or via normal CM tools (eg, salt). That being said, it may occasionally be desirable to create a set of elements which express a distinct configuration of the same software components. In this way, depending on the hardware and in which availability zone it is to be deployed, an image would be composed of: * zero or more driver-elements * one or more service-elements * zero or more config-elements It should be noted that this is merely a naming convention to assist in managing elements. Diskimage-builder is not, and should not be, functionally dependent upon specific element names. diskimage-builder has the ability to retrieve source code for an element and place it into a directory on the target image during the extra-data phase. The default location/branch can then be overridden by the process running diskimage-builder, making it possible to use the same element to track more then one branch of a git repository or to get source for a local cache. See :doc:`../elements/source-repositories/README` for more information. Finding other elements ---------------------- DIB exposes an internal ``$IMAGE_ELEMENT_YAML`` variable which provides elements access to the full set of included elements and their paths. This can be used to process local in-element files across all the elements (``pkg-map`` for example). .. code-block:: python import os import yaml elements = yaml.load(os.getenv('IMAGE_ELEMENT_YAML')) for element, path in elements: ... For elements written in Bash, there is a function ``get_image_element_array`` that can be used to instantiate an associative-array of elements and paths (note arrays can not be exported in bash). .. code-block:: bash # note eval to expand the result of the get function eval declare -A image_elements=($(get_image_element_array)) for i in ${!image_elements[$i]}; do element=$i path=${image_elements[$i]} done Debugging elements ------------------ Export ``break`` to drop to a shell during the image build. Break points can be set either before or after any of the hook points by exporting "break=[before|after]-hook-name". Multiple break points can be specified as a comma-delimited string. Some examples: * ``break=before-block-device-size`` will break before the block device size hooks are called. * ``break=before-pre-install`` will break before the pre-install hooks. * ``break=after-error`` will break after an error during an in target hookpoint. The :doc:`../elements/manifests/README` element will make a range of manifest information generated by other elements available for inspection inside and outside the built image. Environment and command line arguments are captured as described in the documentation and can be useful for debugging. Images are built such that the Linux kernel is instructed not to switch into graphical consoles (i.e. it will not activate KMS). This maximises compatibility with remote console interception hardware, such as HP's iLO. However, you will typically only see kernel messages on the console - init daemons (e.g. upstart) will usually be instructed to output to a serial console so nova's console-log command can function. There is an element in the tripleo-image-elements repository called "remove-serial-console" which will force all boot messages to appear on the main console. Ramdisk images can be debugged at run-time by passing ``troubleshoot`` as a kernel command line argument, or by pressing "t" when an error is reached. This will spawn a shell on the console (this can be extremely useful when network interfaces or disks are not detected correctly). Testing Elements ---------------- An element can have functional tests encapsulated inside the element itself. The tests can be written either as shell or python unit tests. shell """"" In order to create a test case, follow these steps: * Create a directory called ``test-elements`` inside your element. * Inside the test-elements directory, create a directory with the name of your test case. The test case directory should have the same structure as an element. For example:: elements/apt-sources/test-elements/test-case-1 * Assert state during each of the element build phases you would like to test. You can exit 1 to indicate a failure. * To exit early and indicate a success, touch a file ``/tmp/dib-test-should-fail`` in the image chroot, then exit 1. Tests are run with ``tools/run_functests.sh``. Running ``run_functests.sh -l`` will show available tests (the example above would be called ``apt-sources/test-case-1``, for example). Specify your test (or a series of tests as separate arguments) on the command line to run it. If it should not be run as part of the default CI run, you can submit a change with it added to ``DEFAULT_SKIP_TESTS`` in that file. Running the functional tests is time consuming. Multiple parallel jobs can be started by specifying ``-j ``. Each of the jobs uses a lot resources (CPU, disk space, RAM) - therefore the job count must carefully be chosen. python """""" To run functional tests locally, install and start docker, then use the following tox command:: tox -efunc Note that running functional tests requires *sudo* rights, thus you may be asked for your password. To run functional tests for one element, append its name to the command:: tox -efunc ironic-agent Additionally, elements can be tested using python unittests. To create a a python test: * Create a directory called ``tests`` in the element directory. * Create an empty file called ``__init__.py`` to make it into a python package. * Create your test files as ``test\whatever.py``, using regular python test code. To run all the tests use testr - ``testr run``. To run just some tests provide one or more regex filters - tests matching any of them are run - ``testr run apt-proxy``. Third party elements -------------------- Additional elements can be incorporated by setting ``ELEMENTS_PATH``, for example if one were building tripleo-images, the variable would be set like: .. sourcecode:: sh export ELEMENTS_PATH=tripleo-image-elements/elements disk-image-create rhel7 cinder-api Linting ------- You should always run ``bin/dib-lint`` over your elements. It will warn you of common issues. sudo """" Using ``sudo`` outside the chroot environment can cause breakout issues where you accidentally modify parts of the host system. ``dib-lint`` will warn if it sees ``sudo`` calls that do not use the path arguments given to elements running outside the chroot. To disable the error for a call you know is safe, add :: # dib-lint: safe_sudo to the end of the ``sudo`` command line. To disable the check for an entire file, add :: # dib-lint: disable=safe_sudo diskimage-builder-2.11.0/doc/source/developer/dib_lint.rst000066400000000000000000000033241324273057700235400ustar00rootroot00000000000000dib-lint ======== dib-lint provides a way to check for common errors in diskimage-builder elements. To use it, simply run the ``dib-lint`` script in a directory containing an ``elements`` directory. The checks will be run against every file found under ``elements``. The following is a list of what is currently caught by dib-lint: * executable: Ensure any files that begin with #! are executable * indent: Ensure that all source code is using an indent of four spaces * element-deps ordering: Ensure all element-deps files are alphabetized * /bin/bash: Ensure all scripts are using bash explicitly * sete: Ensure all scripts are set -e * setu: Ensure all scripts are set -u * setpipefail: Ensure all scripts are set -o pipefail * dibdebugtrace: Ensure all scripts respect the DIB_DEBUG_TRACE variable * tabindent: Ensure no tabs are used for indentation * newline: Ensure that every file ends with a newline * mddocs: Ensure that only markdown-formatted documentation is used * yaml validation: Ensure that any yaml files in the repo have valid syntax Some of the checks can be omitted, either for an entire project or for an individual file. Project exclusions go in tox.ini, using the following section format:: [dib-lint] ignore=sete setpipefail This will cause the set -e and set -o pipefail checks to be ignored. File-specific exclusions are specified as a comment in the relevant file, using the following format:: # dib-lint: disable=sete setpipefail This will exclude the same tests, but only for the file in which the comment appears. Only some of the checks can be disabled. The ones available for exclusion are: * executable * indent * sete * setu * setpipefail * dibdebugtrace * tabindent * newline * mddocs diskimage-builder-2.11.0/doc/source/developer/index.rst000066400000000000000000000022531324273057700230630ustar00rootroot00000000000000Developer Guide =============== .. toctree:: :maxdepth: 1 design components invocation caches developing_elements dib_lint stable_interfaces This documentation explains how to get started with creating your own disk-image-builder elements as well as some high level concepts for element creation. Quickstart ---------- To get started developing with ``diskimage-builder``, install to a ``virtualenv``:: $ mkdir dib $ cd dib $ virtualenv env $ source env/bin/activate $ git clone https://git.openstack.org/openstack/diskimage-builder $ cd diskimage-builder $ pip install -e . You can now simply use ``disk-image-create`` to start building images and testing your changes. When you are done editing, use ``git review`` to submit changes to the upstream gerrit. Python module documentation --------------------------- For internal documentation on the DIB python components, see the :ref:`modindex` Finding Work ------------ We maintain a list of low-hanging-fruit tags on launchpad: * `https://bugs.launchpad.net/diskimage-builder/+bugs?field.tag=low-hanging-fruit` diskimage-builder-2.11.0/doc/source/developer/invocation.rst000066400000000000000000000043111324273057700241220ustar00rootroot00000000000000Developer Installation ====================== Note that for non-development use you can use distribution packages or install the latest release via ``pip`` in a ``virtualenv``. For development purposes, you can use ``pip -e`` to install the latest git tree checkout into a local development/testing ``virtualenv``, or use ``tox -e venv -- disk-image-create`` to run within a ``tox`` created environment. For example, to create a ``virtualenv`` and install :: $ mkdir dib $ cd dib $ virtualenv env $ source env/bin/activate $ git clone https://git.openstack.org/openstack/diskimage-builder $ cd diskimage-builder $ pip install -e . Invocation ========== The scripts can generally just be run. Options can be set on the command line or by exporting variables to override those present in lib/img-defaults. -h to get help. The image building scripts expect to be able to invoke commands with sudo, so if you want them to run non-interactively, you should either run them as root, with sudo -E, or allow your build user to run any sudo command without password. The variable ``ELEMENTS_PATH`` is a colon (:) separated path list to search for elements. The included ``elements`` tree is used when no path is supplied and is always added to the end of the path if a path is supplied. Earlier elements will override later elements, i.e. with ``ELEMENTS_PATH=foo:bar`` the element ``my-element`` will be chosen from ``foo/my-element`` over ``bar/my-element``, or any in-built element of the same name. By default, the image building scripts will not overwrite existing disk images, allowing you to compare the newly built image with the existing one. To change that behaviour, set the variable ``OVERWRITE_OLD_IMAGE`` to any value that isn't ``0``. If this value is zero then any existing image will be moved before the new image is written to the destination. Setting the variable ``DIB_SHOW_IMAGE_USAGE`` will print out a summarised disk-usage report for the final image of files and directories over 10MiB in size. Setting ``DIB_SHOW_IMAGE_USAGE_FULL`` will show all files and directories. These settings can be useful additions to the logs in automated build situations where debugging image-growth may be important. diskimage-builder-2.11.0/doc/source/developer/stable_interfaces.rst000066400000000000000000000013021324273057700254230ustar00rootroot00000000000000Stable Interfaces ================= diskimage-builder and the elements provide several 'stable' interfaces for both developers and users which we aim to preserve during a major version release. These interfaces consist of: The names and arguments of the executable scripts included with diskimage-builder in the bin directory will remain stable. The environment variables that diskimage-builder provides for elements to use will remain stable. The environment variables documented in each element and the values accepted by these environment variables will remain stable. Required environment variables for an element will not be added. Support for build or target distributions will not be removed. diskimage-builder-2.11.0/doc/source/elements000077700000000000000000000000001324273057700266742../../diskimage_builder/elements/ustar00rootroot00000000000000diskimage-builder-2.11.0/doc/source/elements.rst000066400000000000000000000004201324273057700215750ustar00rootroot00000000000000Elements ======== Elements are found in the subdirectory elements. Each element is in a directory named after the element itself. Elements *should* have a README.rst in the root of the element directory describing what it is for. .. toctree:: :glob: elements/*/* diskimage-builder-2.11.0/doc/source/index.rst000066400000000000000000000032471324273057700211020ustar00rootroot00000000000000Diskimage-builder Documentation =============================== ``diskimage-builder`` is a tool for automatically building customized operating-system images for use in clouds and other environments. It includes support for building images based on many major distributions and can produce cloud-images in all common formats (``qcow2``, ``vhd``, ``raw``, etc), bare metal file-system images and ram-disk images. These images are composed from the many included ``elements``; ``diskimage-builder`` acts as a framework to easily add your own elements for even further customization. ``diskimage-builder`` is used extensively by the `TripleO project `__ and within `OpenStack Infrastructure `__. Code ---- Release notes for the latest and previous versions are available at: * `http://docs.openstack.org/releasenotes/diskimage-builder/ `__ The code is available at: * `https://git.openstack.org/cgit/openstack/diskimage-builder/ `__ Issues ------ Issues are tracked on launchpad at: * ``__ Communication ------------- Communication among the diskimage-builder developers happens on IRC in ``#openstack-dib`` on freenode and on the ``openstack-dev`` mailing list (``openstack-dev@lists.openstack.org``). Table of Contents ----------------- .. toctree:: :maxdepth: 2 user_guide/index developer/index elements specs/README .. # just link this on the dev page .. toctree:: :hidden: api/autoindex diskimage-builder-2.11.0/doc/source/specs/000077500000000000000000000000001324273057700203505ustar00rootroot00000000000000diskimage-builder-2.11.0/doc/source/specs/README.rst000066400000000000000000000063511324273057700220440ustar00rootroot00000000000000================================ diskimage-builder Specifications ================================ Overview ======== This directory is used to hold approved design specifications for changes to the diskimage-builder project. Reviews of the specs are done in gerrit, using a similar workflow to how we review and merge changes to the code itself. For specific policies around specification review, refer to the end of this document. The layout of this directory is:: specs/v/ Where there are two sub-directories: - specs/v/approved: specifications approved but not yet implemented - specs/v/implemented: implemented specifications - specs/v/backlog: unassigned specifications The lifecycle of a specification -------------------------------- Developers proposing a specification should propose a new file in the ``approved`` directory. diskimage-builder-core will review the change in the usual manner for the project, and eventually it will get merged if a consensus is reached. When a specification has been implemented either the developer or someone from diskimage-builder-core will move the implemented specification from the ``approved`` directory to the ``implemented`` directory. It is important to create redirects when this is done so that existing links to the approved specification are not broken. Redirects aren't symbolic links, they are defined in a file which sphinx consumes. An example is at ``specs/v1/redirects``. This directory structure allows you to see what we thought about doing, decided to do, and actually got done. Users interested in functionality in a given release should only refer to the ``implemented`` directory. Example specifications ---------------------- You can find an example spec in :doc:`v1/approved/v1-template` Backlog specifications ---------------------- Additionally, we allow the proposal of specifications that do not have a developer assigned to them. These are proposed for review in the same manner as above, but are added to:: specs/backlog/approved Specifications in this directory indicate the original author has either become unavailable, or has indicated that they are not going to implement the specification. The specifications found here are available as projects for people looking to get involved with diskimage-builder. If you are interested in claiming a spec, start by posting a review for the specification that moves it from this directory to the next active release. Please set yourself as the new `primary assignee` and maintain the original author in the `other contributors` list. Specification review policies ============================= There are some special review policies which diskimage-builder-core will apply when reviewing proposed specifications. They are: Trivial specifications ---------------------- Proposed changes which are trivial (very small amounts of code) and don't change any of our public APIs are sometimes not required to provide a specification. The decision of whether something is trivial or not is a judgement made by the author or by consensus of the project cores, generally trying to err on the side of spec creation. Approved Specifications ======================= .. toctree:: :glob: v1/approved/* diskimage-builder-2.11.0/doc/source/specs/v1/000077500000000000000000000000001324273057700206765ustar00rootroot00000000000000diskimage-builder-2.11.0/doc/source/specs/v1/approved/000077500000000000000000000000001324273057700225165ustar00rootroot00000000000000diskimage-builder-2.11.0/doc/source/specs/v1/approved/block-device-lvl1-partitioning.rst000066400000000000000000000156631324273057700311730ustar00rootroot00000000000000.. This work is licensed under a Creative Commons Attribution 3.0 Unported License. http://creativecommons.org/licenses/by/3.0/legalcode ======================================== Block Device Setup Level 1: Partitioning ======================================== During the creation of a disk image (e.g. for a VM), there is the need to create, setup, configure and afterwards detach some kind of storage where the newly installed OS can be copied to or directly installed in. Remark ------ The implementation for this proposed changed already exists, was discussed and is currently waiting for reviews [1]. To have a complete overview over the block device setup, this document is provided. The dependencies are not implemented as they should be, because * the spec process is currently in the phase of discussion and not finalized [2], * the implementation was finished and reviewed before the spec process was described. [1] Problem description =================== When setting up a block device there is the need to partitioning the block device. Use Cases --------- User (Actor: End User) wants to create multiple partitions in multiple block devices where the new system is installed in. The user wants to specify if the image should be optimized for speed or for size. The user wants the same behavior independently of the current host or target OS. Proposed change =============== Move the partitioning functionality from `elements/vm/block-device.d/10-partition` to a new block_device python module: `level1/partitioning.py`. Instead of using a program or a library, the data is written directly with the help of python `file.write()` into the disk image. Alternatives ------------ The existing implementation uses the `parted` program (old versions of DIB were using `sfdisk`). The first implementations of this change used the python-parted library. All these approaches have a major drawback: they automatically *optimize* based on information collected on the host system - and not of the target system. Therefore the resulting partitioning layout may lead to a degradation of performance on the target system. A change in these external programs and libraries also lead to errors during a DIB run [4] or there are general issues [7]. Also everything build around GNU parted falls under the GPL2 (not LGPL2) license - which is incompatible with the currently used Apache license in diskimage-builder. API impact ---------- Extends the (optional) environment variable ``DIB_BLOCK_DEVICE_CONFIG``: a JSON structure to configure the (complete) block device setup. For this proposal the second entry in the original list will be used (the first part (as described in [5]) is used by the level 0 modules). The name of this module is `partitioning` (element[0]). The value (element[1]) is a dictionary. For each disk that should be partitioned there exists one entry in the dictionary. The key is the name of the disk (see [5] how to specify names for block device level 0). The value is a dictionary that defines the partitioning of each disk. There are the following key / value pairs to define one disk: label (mandatory) Possible values: 'mbr' This uses the Master Boot Record (MBR) layout for the disk. (Later on this can be extended, e.g. using GPT). align (optional - default value '1MiB') Set the alignment of the partition. This must be a multiple of the block size (i.e. 512 bytes). The default of 1MiB (~ 2048 * 512 bytes blocks) is the default for modern systems and known to perform well on a wide range of targets [6]. For each partition there might be some space that is not used - which is `align` - 512 bytes. For the default of 1MiB exactly 1048064 bytes (= 1 MiB - 512 byte) are not used in the partition itself. Please note that if a boot loader should be written to the disk or partition, there is a need for some space. E.g. grub needs 63 * 512 byte blocks between the MBR and the start of the partition data; this means when grub will be installed, the `align` must be set at least to 64 * 512 byte = 32 KiB. partitions (mandatory) A list of dictionaries. Each dictionary describes one partition. The following key / value pairs can be given for each partition: name (mandatory) The name of the partition. With the help of this name, the partition can later be referenced, e.g. while creating a file system. flags (optional) List of flags for the partition. Default: empty. Possible values: boot Sets the boot flag for the partition size (mandatory) The size of the partition. The size can either be an absolute number using units like `10GiB` or `1.75TB` or relative (percentage) numbers: in the later case the size is calculated based on the remaining free space. Example: .. code-block:: yaml ["partitioning", {"rootdisk": { "label": "mbr", "partitions": [{"name": "part-01", "flags": ["boot"], "size": "100%"}]}}] Security impact --------------- None - functionality stays the same. Other end user impact --------------------- None. Performance Impact ------------------ Measurements showed there is a performance degradation for the target system of the partition table is not correctly aligned: writing takes about three times longer on an incorrect aligned system vs. one that is correctly aligned. Implementation ============== Assignee(s) ----------- Primary assignee: ansreas (andreas@florath.net) Work Items ---------- None - this is already a small part of a bigger change [1]. Dependencies ============ None. Testing ======= The refactoring introduces no new test cases: the functionality is tested during each existing test building VM images. Documentation Impact ==================== End user: the additional environment variable is described. References ========== [1] Refactor: block-device handling (partitioning) https://review.openstack.org/322671 [2] Add specs dir https://review.openstack.org/336109 [3] Old implementation using parted-lib https://review.openstack.org/#/c/322671/1..7/elements/block-device/pylib/block-device/level1/Partitioning.py [4] ERROR: embedding is not possible, but this is required for cross-disk install http://lists.openstack.org/pipermail/openstack-dev/2016-June/097789.html [5] Refactor: block-device handling (local loop) https://review.openstack.org/319591 [6] Proper alignment of partitions on an Advanced Format HDD using Parted http://askubuntu.com/questions/201164/proper-alignment-of-partitions-on-an-advanced-format-hdd-using-parted [7] Red Hat Enterprise Linux 6 - Creating a 7TB Partition Using parted Always Shows "The resulting partition is not properly aligned for best performance" http://h20564.www2.hpe.com/hpsc/doc/public/display?docId=emr_na-c03479326&DocLang=en&docLocale=en_US&jumpid=reg_r11944_uken_c-001_title_r0001 [8] Spec for changing the block device handling https://review.openstack.org/336946 diskimage-builder-2.11.0/doc/source/specs/v1/approved/block-device-overview.rst000066400000000000000000000120531324273057700274440ustar00rootroot00000000000000.. This work is licensed under a Creative Commons Attribution 3.0 Unported License. http://creativecommons.org/licenses/by/3.0/legalcode ================== Block Device Setup ================== During the creation of a disk image (e.g. for a VM), there is the need to create, setup, configure and afterwards detach some kind of storage where the newly installed OS can be copied to or directly installed in. Problem description =================== Currently dib is somewhat limited when it comes to setting up the block device: only one partition that can be used for data. LVM, encryption, multi-device or installation in an already existing block device is not supported. In addition there are several places (main, lib, elements) where the current way of handling the block device is used (spread knowledge and implementation). Also it is not possible, to implement the handling as different elements: it is not possible to pass results of one element in the same phase to another element. Passing results from one phase to dib main is limited. Use Cases --------- Possible use cases are (Actor: End User) #. User wants to use an existing block device to install an system image in (like hd, iSCSI, SAN lun, ...). #. User wants that the system will be installed in multiple partitions. #. User wants that the partitioning is done in a specific way (optimize for speed, optimize for size). #. User wants to use LVM to install the system in (multiple PV, VG and LV). #. User wants to encrypt a partition or a LV where (parts) of the system are installed in. #. User wants specific file systems on specific partitions or LVs. Please note that these are only examples and details are described and implemented by different sub-specs. Proposed change =============== Because of the current way to execute elements, it is not possible to have different elements for each feature. Instead the changes will be implemented in a python module 'block_device' placed in the 'diskimage_builder' directory. The entry-point mechanism is used to create callable python programs. These python programs are directly called from within the dib-main. There is the need to implement some functions or classes that take care about common used new functionality: e.g. storing state between phases, calling python sub-modules and passing arguments around. These functionality is implemented as needed - therefore it is most likely that the first patch implements also big parts of these infrastructure tasks. Alternatives ------------ #. Rewrite DIB in the way that elements can interchange data, even if they are called during one phase. This would influence the way all existing elements are called - and might lead to unpredictable results. #. In addition there is the need to introduce at least two additional phases: because major parts of the block device handling are currently done in main and these must be passed over to elements. #. Another way would be to implement everything in one element: this has the disadvantage, that other elements are not allowed to use the 'block_device' phase any longer and also passing around configuration and results is still not possible (see [3]). API impact ---------- Is described in the sub-elements. Security impact --------------- Is described in the sub-elements. Other end user impact --------------------- Paradigm changes from execute script to configuration for block_device phase. Performance Impact ------------------ Is described in the sub-elements. Implementation ============== Assignee(s) ----------- Primary assignee: ansreas (andreas@florath.net) Would be good, if other people would support this - and specify and implement modules. Work Items ---------- This is an overview over changes in the block device layer. Each level or module needs it's own spec. A first step is to reimplement the existing functionality, this contains: #. Level 0: Local Loop module Use loop device on local image file (This is already implemented: [1]) #. Level 1: partitioning module (This is already implemented: [4]) #. Level 2: Create File System An initial module uses ext4 only #. Level 3: Mounting As a second step the following functionality can be added: * Level 1: LVM module * Level 2: Create File System (swap) * Level 2: Create File System (vfat, needed for UEFI) * Level 2: Create File System (xfs) Of course any other functionality can also be added when needed and wanted. Dependencies ============ Is described in the sub-elements. Testing ======= Is described in the sub-elements. Documentation Impact ==================== Is described in the sub-elements. References ========== [1] Implementation of Level 0: Local Loop module https://review.openstack.org/319591 [2] 'Block Device Setup for Disk-Image-Builder' https://etherpad.openstack.org/p/C80jjsAs4x [3] partitioning-parted This was a first try to implement everything as an element - it shows the limitation. https://review.openstack.org/313938 [4] Implementation of Level 1: partitioning module https://review.openstack.org/322671 diskimage-builder-2.11.0/doc/source/specs/v1/approved/v1-template.rst000066400000000000000000000135211324273057700254110ustar00rootroot00000000000000.. This work is licensed under a Creative Commons Attribution 3.0 Unported License. http://creativecommons.org/licenses/by/3.0/legalcode ============================================== Example Spec - The title of your specification ============================================== Introduction paragraph -- why are we doing anything? A single paragraph of prose that operators can understand. The title and this first paragraph should be used as the subject line and body of the commit message respectively. Some notes about the diskimage-bulider spec process: * Not all changes need a spec. For more information see * The aim of this document is first to define the problem we need to solve, and second agree the overall approach to solve that problem. * This is not intended to be extensive documentation for a new feature. * You should aim to get your spec approved before writing your code. While you are free to write prototypes and code before getting your spec approved, its possible that the outcome of the spec review process leads you towards a fundamentally different solution than you first envisaged. * But, API changes are held to a much higher level of scrutiny. As soon as an API change merges, we must assume it could be in production somewhere, and as such, we then need to support that API change forever. To avoid getting that wrong, we do want lots of details about API changes upfront. Some notes about using this template: * Your spec should be in ReSTructured text, like this template. * Please wrap text at 79 columns. * Please do not delete any of the sections in this template. If you have nothing to say for a whole section, just write: None * For help with syntax, see http://sphinx-doc.org/rest.html * If you would like to provide a diagram with your spec, ascii diagrams are required. http://asciiflow.com/ is a very nice tool to assist with making ascii diagrams. The reason for this is that the tool used to review specs is based purely on plain text. Plain text will allow review to proceed without having to look at additional files which can not be viewed in gerrit. It will also allow inline feedback on the diagram itself. Problem description =================== A detailed description of the problem. What problem is this blueprint addressing? Use Cases --------- What use cases does this address? What impact on actors does this change have? Ensure you are clear about the actors in each use case: Developer, End User, etc. Proposed change =============== Here is where you cover the change you propose to make in detail. How do you propose to solve this problem? If this is one part of a larger effort make it clear where this piece ends. In other words, what's the scope of this effort? At this point, if you would like to just get feedback on if the problem and proposed change fit in diskimage-builder, you can stop here and post this for review to get preliminary feedback. If so please say: Posting to get preliminary feedback on the scope of this spec. Alternatives ------------ What other ways could we do this thing? Why aren't we using those? This doesn't have to be a full literature review, but it should demonstrate that thought has been put into why the proposed solution is an appropriate one. API impact ---------- Describe how this will effect our public interfaces. Will this be adding new environment variables? Deprecating existing ones? Adding a new command line argument? Security impact --------------- Describe any potential security impact on the system. Other end user impact --------------------- Aside from the API, are there other ways a user will interact with this feature? Performance Impact ------------------ Describe any potential performance impact on the system, for example how often will new code be called, does it perform any intense processing or data manipulation. Implementation ============== Assignee(s) ----------- Who is leading the writing of the code? Or is this a blueprint where you're throwing it out there to see who picks it up? If more than one person is working on the implementation, please designate the primary author and contact. Primary assignee: Other contributors: Work Items ---------- Work items or tasks -- break the feature up into the things that need to be done to implement it. Those parts might end up being done by different people, but we're mostly trying to understand the timeline for implementation. Dependencies ============ * Include specific references to specs in diskimage-builder or in other projects, that this one either depends on or is related to. * If this requires functionality of another project that is not currently used by diskimage-builder document that fact. Testing ======= Please discuss the important scenarios needed to test here, as well as specific edge cases we should be ensuring work correctly. For each scenario please specify if this requires specialized hardware, or software. Is this untestable in gate given current limitations (specific hardware / software configurations available)? If so, are there mitigation plans (gate enhancements, etc). Documentation Impact ==================== Which audiences are affected most by this change, and which documentation files need to be changed. Do we need to add information about this change to the developer guide, the user guide, certain elements, etc. References ========== Please add any useful references here. You are not required to have any reference. Moreover, this specification should still make sense when your references are unavailable. Examples of what you could include are: * Links to mailing list or IRC discussions * Links to notes from a summit session * Links to relevant research, if appropriate * Related specifications as appropriate * Anything else you feel it is worthwhile to refer to diskimage-builder-2.11.0/doc/source/user_guide/000077500000000000000000000000001324273057700213665ustar00rootroot00000000000000diskimage-builder-2.11.0/doc/source/user_guide/building_an_image.rst000066400000000000000000000470511324273057700255440ustar00rootroot00000000000000Building An Image ================= Now that you have diskimage-builder properly :doc:`installed ` you can get started by building your first disk image. VM Image -------- Our first image is going to be a bootable vm image using one of the standard supported distribution :doc:`elements <../elements>` (Ubuntu or Fedora). The following command will start our image build (distro must be either 'ubuntu' or 'fedora'): :: disk-image-create vm This will create a qcow2 file 'image.qcow2' which can then be booted. Elements -------- It is important to note that we are passing in a list of :doc:`elements <../elements>` to disk-image-create in our above command. Elements are how we decide what goes into our image and what modifications will be performed. Some elements provide a root filesystem, such as the ubuntu or fedora element in our example above, which other elements modify to create our image. At least one of these 'distro elements' must be specified when performing an image build. It's worth pointing out that there are many distro elements (you can even create your own), and even multiples for some of the distros. This is because there are often multiple ways to install a distro which are very different. For example: One distro element might use a cloud image while another uses a package installation tool to build a root filesystem for the same distro. Other elements modify our image in some way. The 'vm' element in our example above ensures that our image has a bootloader properly installed. This is only needed for certain use cases and certain output formats and therefore it is not performed by default. Output Formats -------------- By default a qcow2 image is created by the disk-image-create command. Other output formats may be specified using the `-t ` argument. Multiple output formats can also be specified by comma separation. The supported output formats are: * qcow2 * tar * tgz * squashfs * vhd * docker * raw Disk Image Layout ----------------- The disk image layout (like number of images, partitions, LVM, disk encryption) is something which should be set up during the initial image build: it is mostly not possible to change these things later on. There are currently two defaults: * When using the `vm` element a MBR based partition layout is created with exactly one partition that fills up the whole disk and used as root device. * When not using the `vm` element a plain filesystem image, without any partitioning, is created. The user can overwrite the default handling by setting the environment variable `DIB_BLOCK_DEVICE_CONFIG`. This variable must hold YAML structured configuration data. The default when using the `vm` element is: .. code-block:: yaml DIB_BLOCK_DEVICE_CONFIG=' - local_loop: name: image0 - partitioning: base: image0 label: mbr partitions: - name: root flags: [ boot, primary ] size: 100% mkfs: mount: mount_point: / fstab: options: "defaults" fsck-passno: 1' The default when not using the `vm` element is: .. code-block:: yaml DIB_BLOCK_DEVICE_CONFIG=' - local_loop: name: image0 mkfs: name: mkfs_root mount: mount_point: / fstab: options: "defaults" fsck-passno: 1' There are a lot of different options for the different levels. The following sections describe each level in detail. General Remarks +++++++++++++++ In general each module that depends on another module has a `base` element that points to the depending base. Also each module has a `name` that can be used to reference the module. Tree-Like vs. Complete Digraph Configuration ++++++++++++++++++++++++++++++++++++++++++++ The configuration is specified as a digraph_. Each module is a node; a edge is the relation of the current element to its `base`. Because the general digraph_ approach is somewhat complex when it comes to write it down, the configuration can also be given as a tree_. .. _digraph: https://en.wikipedia.org/wiki/Directed_graph .. _tree: https://en.wikipedia.org/wiki/Tree_(graph_theory) Example: The tree like notation .. code-block:: yaml mkfs: name: root_fs base: root_part mount: mount_point: / is exactly the same as writing .. code-block:: yaml mkfs: name: root_fs base: root_part mount: name: mount_root_fs base: root_fs mount_point: / Non existing `name` and `base` entries in the tree notation are automatically generated: the `name` is the name of the base module prepended by the type-name of the module itself; the `base` element is automatically set to the parent node in the tree. In mostly all cases the much simpler tree notation can be used. Nevertheless there are some use cases when the more general digraph notation is needed. Example: when there is the need to combine two or more modules into one new, like combining a couple of physical volumes into one volume group. Tree and digraph notations can be mixed as needed in a configuration. Limitations +++++++++++ To provide an interface towards the existing elements, there are currently three fixed keys used - which are not configurable: * `root-label`: this is the label of the block device that is mounted at `/`. * `image-block-partition`: if there is a block device with the name `root` this is used else the block device with the name `image0` is used. * `image-path`: the path of the image that contains the root file system is taken from the `image0`. Level 0 +++++++ Module: Local Loop .................. This module generates a local image file and uses the loop device to create a block device from it. The symbolic name for this module is `local_loop`. Configuration options: name (mandatory) The name of the image. This is used as the name for the image in the file system and also as a symbolic name to be able to reference this image (e.g. to create a partition table on this disk). size (optional) The size of the disk. The size can be expressed using unit names like TiB (1024^4 bytes) or GB (1000^3 bytes). Examples: 2.5GiB, 12KB. If the size is not specified here, the size as given to disk-image-create (--image-size) or the automatically computed size is used. directory (optional) The directory where the image is created. Example: .. code-block:: yaml local_loop: name: image0 local_loop: name: data_image size: 7.5GiB directory: /var/tmp This creates two image files and uses the loop device to use them as block devices. One image file called `image0` is created with default size in the default temp directory. The second image has the size of 7.5GiB and is created in the `/var/tmp` folder. Level 1 +++++++ Module: Partitioning .................... This module generates partitions on existing block devices. This means that it is possible to take any kind of block device (e.g. LVM, encrypted, ...) and create partition information in it. The symbolic name for this module is `partitioning`. Currently the only supported partitioning layout is Master Boot Record `MBR`. It is possible to create primary or logical partitions or a mix of them. The numbering of the primary partitions will start at 1, e.g. `/dev/vda1`; logical partitions will typically start with `5`, e.g. `/dev/vda5` for the first partition, `/dev/vda6` for the second and so on. The number of logical partitions created by this module is theoretical unlimited and it was tested with more than 1000 partitions inside one block device. Nevertheless the Linux kernel and different tools (like `parted`, `sfdisk`, `fdisk`) have some default maximum number of partitions that they can handle. Please consult the documentation of the appropriate software you plan to use and adapt the number of partitions. Partitions are created in the order they are configured. Primary partitions - if needed - must be first in the list. There are the following key / value pairs to define one partition table: base (mandatory) The base device where to create the partitions in. label (mandatory) Possible values: 'mbr' This uses the Master Boot Record (MBR) layout for the disk. (There are currently plans to add GPT later on.) align (optional - default value '1MiB') Set the alignment of the partition. This must be a multiple of the block size (i.e. 512 bytes). The default of 1MiB (~ 2048 * 512 bytes blocks) is the default for modern systems and known to perform well on a wide range of targets. For each partition there might be some space that is not used - which is `align` - 512 bytes. For the default of 1MiB exactly 1048064 bytes (= 1 MiB - 512 byte) are not used in the partition itself. Please note that if a boot loader should be written to the disk or partition, there is a need for some space. E.g. grub needs 63 * 512 byte blocks between the MBR and the start of the partition data; this means when grub will be installed, the `align` must be set at least to 64 * 512 byte = 32 KiB. partitions (mandatory) A list of dictionaries. Each dictionary describes one partition. The following key / value pairs can be given for each partition: name (mandatory) The name of the partition. With the help of this name, the partition can later be referenced, e.g. when creating a file system. flags (optional) List of flags for the partition. Default: empty. Possible values: boot Sets the boot flag for the partition primary Partition should be a primary partition. If not set a logical partition will be created. size (mandatory) The size of the partition. The size can either be an absolute number using units like `10GiB` or `1.75TB` or relative (percentage) numbers: in the later case the size is calculated based on the remaining free space. type (optional) The partition type stored in the MBR partition table entry. The default value is '0x83' (Linux Default partition). Any valid one byte hexadecimal value may be specified here. Example: .. code-block:: yaml - partitioning: base: image0 label: mbr partitions: - name: part-01 flags: [ boot ] size: 1GiB - name: part-02 size: 100% - partitioning: base: data_image label: mbr partitions: - name: data0 size: 33% - name: data1 size: 50% - name: data2 size: 100% On the `image0` two partitions are created. The size of the first is 1GiB, the second uses the remaining free space. On the `data_image` three partitions are created: all are about 1/3 of the disk size. Module: Lvm ··········· This module generates volumes on existing block devices. This means that it is possible to take any previous created partition, and create volumes information in it. The symbolic name for this module is `lvm`. There are the following key / value pairs to define one set of volumes: pvs (mandatory) A list of dictionaries. Each dictionary describes one physical volume. vgs (mandatory) A list of dictionaries. Each dictionary describes one volume group. lvs (mandatory) A list of dictionaries. Each dictionary describes one logical volume. The following key / value pairs can be given for each `pvs`: name (mandatory) The name of the physical volume. With the help of this name, the physical volume can later be referenced, e.g. when creating a volume group. base (mandatory) The name of the partition where the physical volume needs to be created. options (optional) List of options for the physical volume. It can contain any option supported by the `pvcreate` command. The following key / value pairs can be given for each `vgs`: name (mandatory) The name of the volume group. With the help of this name, the volume group can later be referenced, e.g. when creating a logical volume. base (mandatory) The name(s) of the physical volumes where the volume groups needs to be created. As a volume group can be created on one or more physical volumes, this needs to be a list. options (optional) List of options for the volume group. It can contain any option supported by the `vgcreate` command. The following key / value pairs can be given for each `lvs`: name (mandatory) The name of the logical volume. With the help of this name, the logical volume can later be referenced, e.g. when creating a filesystem. base (mandatory) The name of the volume group where the logical volume needs to be created. size (optional) The exact size of the volume to be created. It accepts the same syntax as the -L flag of the `lvcreate` command. extents (optional) The relative size in extents of the volume to be created. It accepts the same syntax as the -l flag of the `lvcreate` command. Either size or extents need to be passed on the volume creation. options (optional) List of options for the logical volume. It can contain any option supported by the `lvcreate` command. Example: .. code-block:: yaml - lvm: name: lvm pvs: - name: pv options: ["--force"] device: root vgs: - name: vg base: ["pv"] options: ["--force"] lvs: - name: lv_root base: vg size: 1800M - name: lv_tmp base: vg size: 100M - name: lv_var base: vg size: 500M - name: lv_log base: vg size: 100M - name: lv_audit base: vg size: 100M - name: lv_home base: vg size: 200M On the `root` partition a physical volume is created. On that physical volume, a volume group is created. On top of this volume group, six logical volumes are created. Please note that in order to build images that are bootable using volumes, your ramdisk image will need to have that support. If the image you are using does not have it, you can add the needed modules and regenerate it, by including the `dracut-regenerate` element when building it. Level 2 +++++++ Module: Mkfs ............ This module creates file systems on the block device given as `base`. The following key / value pairs can be given: base (mandatory) The name of the block device where the filesystem will be created on. name (mandatory) The name of the partition. This can be used to reference (e.g. mounting) the filesystem. type (mandatory) The type of the filesystem, like `ext4` or `xfs`. label (optional - defaults to the name) The label of the filesystem. This can be used e.g. by grub or in the fstab. opts (optional - defaults to empty list) Options that will passed to the mkfs command. uuid (optional - no default / not used if not givem) The UUID of the filesystem. Not all file systems might support this. Currently there is support for `ext2`, `ext3`, `ext4` and `xfs`. Example: .. code-block:: yaml - mkfs: name: mkfs_root base: root type: ext4 label: cloudimage-root uuid: b733f302-0336-49c0-85f2-38ca109e8bdb opts: "-i 16384" Level 3 +++++++ Module: Mount ............. This module mounts a filesystem. The options are: base (mandatory) The name of the filesystem that will be mounted. name (mandatory) The name of the mount point. This can be used for reference the mount (e.g. creating the fstab). mount_point (mandatory) The mount point of the filesystem. There is no need to list the mount points in the correct order: an algorithm will automatically detect the mount order. Example: .. code-block:: yaml - mount: name: root_mnt base: mkfs_root mount_point: / Level 4 +++++++ Module: fstab ............. This module creates fstab entries. The following options exists. For details please consult the fstab man page. base (mandatory) The name of the mount point that will be written to fstab. name (mandatory) The name of the fstab entry. This can be used later on as reference - and is currently unused. options (optional, defaults to `default`) Special mount options can be given. This is used as the fourth field in the fstab entry. dump-freq (optional, defaults to 0 - don't dump) This is passed to dump to determine which filesystem should be dumped. This is used as the fifth field in the fstab entry. fsck-passno (optional, defaults to 2) Determines the order to run fsck. Please note that this should be set to 1 for the root file system. This is used as the sixth field in the fstab entry. Example: .. code-block:: yaml - fstab: name: var_log_fstab base: var_log_mnt options: nodev,nosuid dump-freq: 2 Filesystem Caveat ----------------- By default, disk-image-create uses a 4k byte-to-inode ratio when creating the filesystem in the image. This allows large 'whole-system' images to utilize several TB disks without exhausting inodes. In contrast, when creating images intended for tenant instances, this ratio consumes more disk space than an end-user would expect (e.g. a 50GB root disk has 47GB avail.). If the image is intended to run within a tens to hundrededs of gigabyte disk, setting the byte-to-inode ratio to the ext4 default of 16k will allow for more usable space on the instance. The default can be overridden by passing ``--mkfs-options`` like this:: disk-image-create --mkfs-options '-i 16384' vm You can also select a different filesystem by setting the ``FS_TYPE`` environment variable. Note ``--mkfs-options`` are options passed to the mfks *driver*, rather than ``mkfs`` itself (i.e. after the initial `-t` argument). Speedups -------- If you have 4GB of available physical RAM (as reported by /proc/meminfo MemTotal), or more, diskimage-builder will create a tmpfs mount to build the image in. This will improve image build time by building it in RAM. By default, the tmpfs file system uses 50% of the available RAM. Therefore, the RAM should be at least the double of the minimum tmpfs size required. For larger images, when no sufficient amount of RAM is available, tmpfs can be disabled completely by passing --no-tmpfs to disk-image-create. ramdisk-image-create builds a regular image and then within that image creates ramdisk. If tmpfs is not used, you will need enough room in /tmp to store two uncompressed cloud images. If tmpfs is used, you would still need /tmp space for one uncompressed cloud image and about 20% of that image for working files. Chosing an Architecture ----------------------- If needed you can specify an override the architecture selection by passing a ``-a`` argument like: :: disk-image-create -a ... Notes about PowerPC Architectures +++++++++++++++++++++++++++++++++ PowerPC can operate in either Big or Little Endian mode. ``ppc64`` always refers to Big Endian operation. When running in little endian mode it can be referred to as ``ppc64le`` or ``ppc64el``. Typically ``ppc64el`` refers to a ``.deb`` based distribution architecture, and ``ppc64le`` refers to a ``.rpm`` based distribution. Regardless of the distribution the kernel architecture is always ``ppc64le``. Notes about s390x (z Systems) Architecture ++++++++++++++++++++++++++++++++++++++++++ Images for s390x can only be build on s390x hosts. Trying to build it with the architecture override on other architecture will cause the build to fail. diskimage-builder-2.11.0/doc/source/user_guide/index.rst000066400000000000000000000002001324273057700232170ustar00rootroot00000000000000User Guide ========== .. toctree:: :maxdepth: 1 supported_distros installation building_an_image install_types diskimage-builder-2.11.0/doc/source/user_guide/install_types.rst000066400000000000000000000033761324273057700250230ustar00rootroot00000000000000Install Types ============= Install types permit elements to be installed from different sources, such as git repositories, distribution packages, or pip. The default install type is 'source' but it can be modified on the disk-image-create command line via the --install-type option. For example you can set: --install-type=package to enable package installs by default. Alternately, you can also set DIB\_DEFAULT\_INSTALLTYPE. Many elements expose different install types. The different implementations live under `--install` directories under an element's install.d. The base element enables the chosen install type by symlinking the correct hook scripts under install.d directly. `` can be a string of alphanumeric and '-' characters, but typically corresponds to the element name. For example, the nova element would provide: nova/install.d/nova-package-install/74-nova nova/install.d/nova-source-install/74-nova The following symlink would be created for the package install type: install.d/74-nova -> nova-package-install/74-nova Or, for the source install type: install.d/74-nova -> nova-source-install/74-nova All other scripts that exist under install.d for an element will be executed as normal. This allows common install code to live in a script under install.d. To set the install type for an element define an environment variable `DIB_INSTALLTYPE_`. Note that if you used `-` characters in your install directory prefix, those need to be replaced with `_` in the environment variable. For example, to enable the package install type for the set of nova elements that use `nova` as the install type prefix, define the following: export DIB_INSTALLTYPE_nova=package diskimage-builder-2.11.0/doc/source/user_guide/installation.rst000066400000000000000000000026241324273057700246250ustar00rootroot00000000000000Installation ============ If your distribution does not proivde packages, you should install ``diskimage-builder`` via ``pip``, mostly likely in a ``virtualenv`` to keep it separate. For example, to create a ``virtualenv`` and install from ``pip`` :: virtualenv ~/dib-virtualenv . ~/dib-virtualenv/bin/activate pip install diskimage-builder Once installed, you will be able to :doc:`build images ` using ``disk-image-create`` and the elements included in the main ``diskimage-builder`` repository. Requirements ------------ Most image formats require the ``qemu-img`` tool which is provided by the ``qemu-utils`` package on Ubuntu/Debian or the ``qemu`` package on Fedora/RHEL/opensuse/Gentoo. When generating images with partitions, the ``kpartx`` tool is needed, which is provided by the ``kpartx`` package. Some image formats, such as ``VHD``, may require additional tools. Please see the ``disk-image-create`` help output for more information. Individual elements can also have additional dependencies for the build host. It is recommended you check the documentation for each element you are using to determine if there are any additional dependencies. Of particular note is the need for the `dev-python/pyyaml` package on Gentoo hosts. Package Installation -------------------- On Gentoo you can emerge diskimage-builder directly. :: emerge app-emulation/diskimage-builder diskimage-builder-2.11.0/doc/source/user_guide/supported_distros.rst000066400000000000000000000007511324273057700257170ustar00rootroot00000000000000Supported Distributions ======================= Distributions which are supported as a build host: - Centos 6, 7 - Debian 8 ("jessie") - Fedora 26, 27 - RHEL 6, 7 - Ubuntu 14.04 ("trusty") - Gentoo - openSUSE Leap 42.2, 42.3 and Tumbleweed Distributions which are supported as a target for an image: - Centos 6, 7 - Debian 8 ("jessie") - Fedora 26, 27 - RHEL 6, 7 - Ubuntu 12.04 ("precise"), 14.04 ("trusty") - Gentoo - openSUSE Leap 42.2, 42.3 and Tumbleweed (opensuse-minimal only) diskimage-builder-2.11.0/playbooks/000077500000000000000000000000001324273057700171715ustar00rootroot00000000000000diskimage-builder-2.11.0/playbooks/legacy/000077500000000000000000000000001324273057700204355ustar00rootroot00000000000000diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-centos-7-extras/000077500000000000000000000000001324273057700305205ustar00rootroot00000000000000diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-centos-7-extras/post.yaml000066400000000000000000000006331324273057700323730ustar00rootroot00000000000000- hosts: primary tasks: - name: Copy files from {{ ansible_user_dir }}/workspace/ on node synchronize: src: '{{ ansible_user_dir }}/workspace/' dest: '{{ zuul.executor.log_root }}' mode: pull copy_links: true verify_host: true rsync_opts: - --include=/logs/** - --include=*/ - --exclude=* - --prune-empty-dirs diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-centos-7-extras/run.yaml000066400000000000000000000051201324273057700322060ustar00rootroot00000000000000- hosts: all name: Autoconverted job legacy-dib-dsvm-functests-python2-centos-7-extras from old job gate-dib-dsvm-functests-python2-centos-7-extras-nv tasks: - name: Ensure legacy workspace directory file: path: '{{ ansible_user_dir }}/workspace' state: directory - shell: cmd: | set -u set -e set -x cd ~ /usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git \ git://git.openstack.org \ openstack/diskimage-builder \ openstack/requirements executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -e set -x cd ~/openstack/diskimage-builder /usr/local/jenkins/slave_scripts/install-distro-packages.sh executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -u set -e set -x # # We do things that might exceed the default 2GiB tmpfs, and # use the larger mounted space. cd ~ sudo mkdir /opt/dib_cache sudo chown zuul:zuul /opt/dib_cache export DIB_NO_TMPFS=1 export TMPDIR=/opt/dib_cache virtualenv -p python2 env export UPPER_CONSTRAINTS_FILE=$(pwd)/openstack/requirements/upper-constraints.txt sed -i '/^diskimage-builder/d' $UPPER_CONSTRAINTS_FILE ./env/bin/pip install $(pwd)/openstack/diskimage-builder -c $UPPER_CONSTRAINTS_FILE # TODO(pabelanger): Remove once we migrated to bindep ./openstack/diskimage-builder/tests/install_test_deps.sh # This sets up some repo files pointing to the infra mirrors # which are used during test runs ./openstack/diskimage-builder/contrib/setup-gate-mirrors.sh # UPPER_CONSTRAINTS_FILE is causing side effects when doing image # build, unset it unset UPPER_CONSTRAINTS_FILE # activate the virtualenv so that any tools run by dib run # using the python inside it set +u source ./env/bin/activate set -u ./openstack/diskimage-builder/tests/run_functests.sh \ gentoo/build-succeeds \ debian-minimal/stable-vm set +u deactivate set -u executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-centos-7-image/000077500000000000000000000000001324273057700302745ustar00rootroot00000000000000diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-centos-7-image/post.yaml000066400000000000000000000006331324273057700321470ustar00rootroot00000000000000- hosts: primary tasks: - name: Copy files from {{ ansible_user_dir }}/workspace/ on node synchronize: src: '{{ ansible_user_dir }}/workspace/' dest: '{{ zuul.executor.log_root }}' mode: pull copy_links: true verify_host: true rsync_opts: - --include=/logs/** - --include=*/ - --exclude=* - --prune-empty-dirs diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-centos-7-image/run.yaml000066400000000000000000000053711324273057700317720ustar00rootroot00000000000000- hosts: all name: Autoconverted job legacy-dib-dsvm-functests-python2-centos-7-image from old job gate-dib-dsvm-functests-python2-centos-7-image-nv tasks: - name: Ensure legacy workspace directory file: path: '{{ ansible_user_dir }}/workspace' state: directory - shell: cmd: | set -u set -e set -x cd ~ /usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git \ git://git.openstack.org \ openstack/diskimage-builder \ openstack/requirements executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -e set -x cd ~/openstack/diskimage-builder /usr/local/jenkins/slave_scripts/install-distro-packages.sh executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -u set -e set -x # # We do things that might exceed the default 2GiB tmpfs, and # use the larger mounted space. cd ~ sudo mkdir /opt/dib_cache sudo chown zuul:zuul /opt/dib_cache export DIB_NO_TMPFS=1 export TMPDIR=/opt/dib_cache virtualenv -p python2 env export UPPER_CONSTRAINTS_FILE=$(pwd)/openstack/requirements/upper-constraints.txt sed -i '/^diskimage-builder/d' $UPPER_CONSTRAINTS_FILE ./env/bin/pip install $(pwd)/openstack/diskimage-builder -c $UPPER_CONSTRAINTS_FILE # TODO(pabelanger): Remove once we migrated to bindep ./openstack/diskimage-builder/tests/install_test_deps.sh # This sets up some repo files pointing to the infra mirrors # which are used during test runs ./openstack/diskimage-builder/contrib/setup-gate-mirrors.sh # UPPER_CONSTRAINTS_FILE is causing side effects when doing image # build, unset it unset UPPER_CONSTRAINTS_FILE # activate the virtualenv so that any tools run by dib run # using the python inside it set +u source ./env/bin/activate set -u ./openstack/diskimage-builder/tests/run_functests.sh \ centos7/build-succeeds \ opensuse/build-succeeds \ opensuse/opensuse423-build-succeeds \ fedora/build-succeeds \ ubuntu/trusty-build-succeeds \ ubuntu/xenial-build-succeeds set +u deactivate set -u executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-centos-7/000077500000000000000000000000001324273057700272145ustar00rootroot00000000000000diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-centos-7/post.yaml000066400000000000000000000006331324273057700310670ustar00rootroot00000000000000- hosts: primary tasks: - name: Copy files from {{ ansible_user_dir }}/workspace/ on node synchronize: src: '{{ ansible_user_dir }}/workspace/' dest: '{{ zuul.executor.log_root }}' mode: pull copy_links: true verify_host: true rsync_opts: - --include=/logs/** - --include=*/ - --exclude=* - --prune-empty-dirs diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-centos-7/run.yaml000066400000000000000000000047621324273057700307150ustar00rootroot00000000000000- hosts: all name: Autoconverted job legacy-dib-dsvm-functests-python2-centos-7 from old job gate-dib-dsvm-functests-python2-centos-7 tasks: - name: Ensure legacy workspace directory file: path: '{{ ansible_user_dir }}/workspace' state: directory - shell: cmd: | set -u set -e set -x cd ~ /usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git \ git://git.openstack.org \ openstack/diskimage-builder \ openstack/requirements executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -e set -x cd ~/openstack/diskimage-builder /usr/local/jenkins/slave_scripts/install-distro-packages.sh executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -u set -e set -x # # We do things that might exceed the default 2GiB tmpfs, and # use the larger mounted space. cd ~ sudo mkdir /opt/dib_cache sudo chown zuul:zuul /opt/dib_cache export DIB_NO_TMPFS=1 export TMPDIR=/opt/dib_cache virtualenv -p python2 env export UPPER_CONSTRAINTS_FILE=$(pwd)/openstack/requirements/upper-constraints.txt sed -i '/^diskimage-builder/d' $UPPER_CONSTRAINTS_FILE ./env/bin/pip install $(pwd)/openstack/diskimage-builder -c $UPPER_CONSTRAINTS_FILE # UPPER_CONSTRAINTS_FILE is causing side effects when doing image # build, unset it unset UPPER_CONSTRAINTS_FILE # TODO(pabelanger): Remove once we migrated to bindep ./openstack/diskimage-builder/tests/install_test_deps.sh # This sets up some repo files pointing to the infra mirrors # which are used during test runs ./openstack/diskimage-builder/contrib/setup-gate-mirrors.sh # activate the virtualenv so that any tools run by dib run # using the python inside it set +u source ./env/bin/activate set -u ./openstack/diskimage-builder/tests/run_functests.sh set +u deactivate set -u executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-opensuse-423/000077500000000000000000000000001324273057700277245ustar00rootroot00000000000000diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-opensuse-423/post.yaml000066400000000000000000000006331324273057700315770ustar00rootroot00000000000000- hosts: primary tasks: - name: Copy files from {{ ansible_user_dir }}/workspace/ on node synchronize: src: '{{ ansible_user_dir }}/workspace/' dest: '{{ zuul.executor.log_root }}' mode: pull copy_links: true verify_host: true rsync_opts: - --include=/logs/** - --include=*/ - --exclude=* - --prune-empty-dirs diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-opensuse-423/run.yaml000066400000000000000000000050001324273057700314070ustar00rootroot00000000000000- hosts: all name: Autoconverted job legacy-dib-dsvm-functests-python2-opensuse-423 from old job gate-dib-dsvm-functests-python2-opensuse-423-nv tasks: - name: Ensure legacy workspace directory file: path: '{{ ansible_user_dir }}/workspace' state: directory - shell: cmd: | set -u set -e set -x cd ~ /usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git \ git://git.openstack.org \ openstack/diskimage-builder \ openstack/requirements executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -e set -x cd ~/openstack/diskimage-builder /usr/local/jenkins/slave_scripts/install-distro-packages.sh executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -u set -e set -x # # We do things that might exceed the default 2GiB tmpfs, and # use the larger mounted space. cd ~ sudo mkdir /opt/dib_cache sudo chown zuul:zuul /opt/dib_cache export DIB_NO_TMPFS=1 export TMPDIR=/opt/dib_cache virtualenv -p python2 env export UPPER_CONSTRAINTS_FILE=$(pwd)/openstack/requirements/upper-constraints.txt sed -i '/^diskimage-builder/d' $UPPER_CONSTRAINTS_FILE ./env/bin/pip install $(pwd)/openstack/diskimage-builder -c $UPPER_CONSTRAINTS_FILE # TODO(pabelanger): Remove once we migrated to bindep ./openstack/diskimage-builder/tests/install_test_deps.sh # This sets up some repo files pointing to the infra mirrors # which are used during test runs ./openstack/diskimage-builder/contrib/setup-gate-mirrors.sh # UPPER_CONSTRAINTS_FILE is causing side effects when doing image # build, unset it unset UPPER_CONSTRAINTS_FILE # activate the virtualenv so that any tools run by dib run # using the python inside it set +u source ./env/bin/activate set -u ./openstack/diskimage-builder/tests/run_functests.sh set +u deactivate set -u executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-extras/000077500000000000000000000000001324273057700317535ustar00rootroot00000000000000diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-extras/post.yaml000066400000000000000000000006331324273057700336260ustar00rootroot00000000000000- hosts: primary tasks: - name: Copy files from {{ ansible_user_dir }}/workspace/ on node synchronize: src: '{{ ansible_user_dir }}/workspace/' dest: '{{ zuul.executor.log_root }}' mode: pull copy_links: true verify_host: true rsync_opts: - --include=/logs/** - --include=*/ - --exclude=* - --prune-empty-dirs diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-extras/run.yaml000066400000000000000000000051321324273057700334440ustar00rootroot00000000000000- hosts: all name: Autoconverted job legacy-dib-dsvm-functests-python2-ubuntu-trusty-extras from old job gate-dib-dsvm-functests-python2-ubuntu-trusty-extras-nv tasks: - name: Ensure legacy workspace directory file: path: '{{ ansible_user_dir }}/workspace' state: directory - shell: cmd: | set -u set -e set -x cd ~ /usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git \ git://git.openstack.org \ openstack/diskimage-builder \ openstack/requirements executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -e set -x cd ~/openstack/diskimage-builder /usr/local/jenkins/slave_scripts/install-distro-packages.sh executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -u set -e set -x # # We do things that might exceed the default 2GiB tmpfs, and # use the larger mounted space. cd ~ sudo mkdir /opt/dib_cache sudo chown zuul:zuul /opt/dib_cache export DIB_NO_TMPFS=1 export TMPDIR=/opt/dib_cache virtualenv -p python2 env export UPPER_CONSTRAINTS_FILE=$(pwd)/openstack/requirements/upper-constraints.txt sed -i '/^diskimage-builder/d' $UPPER_CONSTRAINTS_FILE ./env/bin/pip install $(pwd)/openstack/diskimage-builder -c $UPPER_CONSTRAINTS_FILE # TODO(pabelanger): Remove once we migrated to bindep ./openstack/diskimage-builder/tests/install_test_deps.sh # This sets up some repo files pointing to the infra mirrors # which are used during test runs ./openstack/diskimage-builder/contrib/setup-gate-mirrors.sh # UPPER_CONSTRAINTS_FILE is causing side effects when doing image # build, unset it unset UPPER_CONSTRAINTS_FILE # activate the virtualenv so that any tools run by dib run # using the python inside it set +u source ./env/bin/activate set -u ./openstack/diskimage-builder/tests/run_functests.sh \ gentoo/build-succeeds \ debian-minimal/stable-vm set +u deactivate set -u executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-image/000077500000000000000000000000001324273057700315275ustar00rootroot00000000000000diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-image/post.yaml000066400000000000000000000006331324273057700334020ustar00rootroot00000000000000- hosts: primary tasks: - name: Copy files from {{ ansible_user_dir }}/workspace/ on node synchronize: src: '{{ ansible_user_dir }}/workspace/' dest: '{{ zuul.executor.log_root }}' mode: pull copy_links: true verify_host: true rsync_opts: - --include=/logs/** - --include=*/ - --exclude=* - --prune-empty-dirs diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-image/run.yaml000066400000000000000000000054041324273057700332220ustar00rootroot00000000000000- hosts: all name: Autoconverted job legacy-dib-dsvm-functests-python2-ubuntu-trusty-image from old job gate-dib-dsvm-functests-python2-ubuntu-trusty-image-nv tasks: - name: Ensure legacy workspace directory file: path: '{{ ansible_user_dir }}/workspace' state: directory - shell: cmd: | set -u set -e set -x cd ~ /usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git \ git://git.openstack.org \ openstack/diskimage-builder \ openstack/requirements executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -e set -x cd ~/openstack/diskimage-builder /usr/local/jenkins/slave_scripts/install-distro-packages.sh executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -u set -e set -x # # We do things that might exceed the default 2GiB tmpfs, and # use the larger mounted space. cd ~ sudo mkdir /opt/dib_cache sudo chown zuul:zuul /opt/dib_cache export DIB_NO_TMPFS=1 export TMPDIR=/opt/dib_cache virtualenv -p python2 env export UPPER_CONSTRAINTS_FILE=$(pwd)/openstack/requirements/upper-constraints.txt sed -i '/^diskimage-builder/d' $UPPER_CONSTRAINTS_FILE ./env/bin/pip install $(pwd)/openstack/diskimage-builder -c $UPPER_CONSTRAINTS_FILE # TODO(pabelanger): Remove once we migrated to bindep ./openstack/diskimage-builder/tests/install_test_deps.sh # This sets up some repo files pointing to the infra mirrors # which are used during test runs ./openstack/diskimage-builder/contrib/setup-gate-mirrors.sh # UPPER_CONSTRAINTS_FILE is causing side effects when doing image # build, unset it unset UPPER_CONSTRAINTS_FILE # activate the virtualenv so that any tools run by dib run # using the python inside it set +u source ./env/bin/activate set -u ./openstack/diskimage-builder/tests/run_functests.sh \ centos7/build-succeeds \ opensuse/build-succeeds \ opensuse/opensuse423-build-succeeds \ fedora/build-succeeds \ ubuntu/trusty-build-succeeds \ ubuntu/xenial-build-succeeds set +u deactivate set -u executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty/000077500000000000000000000000001324273057700304475ustar00rootroot00000000000000diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty/post.yaml000066400000000000000000000006331324273057700323220ustar00rootroot00000000000000- hosts: primary tasks: - name: Copy files from {{ ansible_user_dir }}/workspace/ on node synchronize: src: '{{ ansible_user_dir }}/workspace/' dest: '{{ zuul.executor.log_root }}' mode: pull copy_links: true verify_host: true rsync_opts: - --include=/logs/** - --include=*/ - --exclude=* - --prune-empty-dirs diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty/run.yaml000066400000000000000000000047771324273057700321560ustar00rootroot00000000000000- hosts: all name: Autoconverted job legacy-dib-dsvm-functests-python2-ubuntu-trusty from old job gate-dib-dsvm-functests-python2-ubuntu-trusty tasks: - name: Ensure legacy workspace directory file: path: '{{ ansible_user_dir }}/workspace' state: directory - shell: cmd: | set -u set -e set -x cd ~ /usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git \ git://git.openstack.org \ openstack/diskimage-builder \ openstack/requirements executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -e set -x cd ~/openstack/diskimage-builder /usr/local/jenkins/slave_scripts/install-distro-packages.sh executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -u set -e set -x # # We do things that might exceed the default 2GiB tmpfs, and # use the larger mounted space. cd ~ sudo mkdir /opt/dib_cache sudo chown zuul:zuul /opt/dib_cache export DIB_NO_TMPFS=1 export TMPDIR=/opt/dib_cache virtualenv -p python2 env export UPPER_CONSTRAINTS_FILE=$(pwd)/openstack/requirements/upper-constraints.txt sed -i '/^diskimage-builder/d' $UPPER_CONSTRAINTS_FILE ./env/bin/pip install $(pwd)/openstack/diskimage-builder -c $UPPER_CONSTRAINTS_FILE # TODO(pabelanger): Remove once we migrated to bindep ./openstack/diskimage-builder/tests/install_test_deps.sh # This sets up some repo files pointing to the infra mirrors # which are used during test runs ./openstack/diskimage-builder/contrib/setup-gate-mirrors.sh # UPPER_CONSTRAINTS_FILE is causing side effects when doing image # build, unset it unset UPPER_CONSTRAINTS_FILE # activate the virtualenv so that any tools run by dib run # using the python inside it set +u source ./env/bin/activate set -u ./openstack/diskimage-builder/tests/run_functests.sh set +u deactivate set -u executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python3-extras/000077500000000000000000000000001324273057700270645ustar00rootroot00000000000000diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python3-extras/post.yaml000066400000000000000000000006331324273057700307370ustar00rootroot00000000000000- hosts: primary tasks: - name: Copy files from {{ ansible_user_dir }}/workspace/ on node synchronize: src: '{{ ansible_user_dir }}/workspace/' dest: '{{ zuul.executor.log_root }}' mode: pull copy_links: true verify_host: true rsync_opts: - --include=/logs/** - --include=*/ - --exclude=* - --prune-empty-dirs diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python3-extras/run.yaml000066400000000000000000000052021324273057700305530ustar00rootroot00000000000000- hosts: all name: Autoconverted job legacy-dib-dsvm-functests-python3-extras from old job gate-dib-dsvm-functests-python3-ubuntu-xenial-extras-nv tasks: - name: Ensure legacy workspace directory file: path: '{{ ansible_user_dir }}/workspace' state: directory - shell: cmd: | set -u set -e set -x cd ~ /usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git \ git://git.openstack.org \ openstack/diskimage-builder \ openstack/requirements executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -e set -x cd ~/openstack/diskimage-builder /usr/local/jenkins/slave_scripts/install-distro-packages.sh executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -u set -e set -x # # We do things that might exceed the default 2GiB tmpfs, and # use the larger mounted space. cd ~ sudo mkdir /opt/dib_cache sudo chown zuul:zuul /opt/dib_cache export DIB_NO_TMPFS=1 export TMPDIR=/opt/dib_cache virtualenv -p python3 env export UPPER_CONSTRAINTS_FILE=$(pwd)/openstack/requirements/upper-constraints.txt sed -i '/^diskimage-builder/d' $UPPER_CONSTRAINTS_FILE ./env/bin/pip install $(pwd)/openstack/diskimage-builder -c $UPPER_CONSTRAINTS_FILE # TODO(pabelanger): Remove once we migrated to bindep ./openstack/diskimage-builder/tests/install_test_deps.sh # This sets up some repo files pointing to the infra mirrors # which are used during test runs ./openstack/diskimage-builder/contrib/setup-gate-mirrors.sh # UPPER_CONSTRAINTS_FILE is causing side effects when doing image # build, unset it unset UPPER_CONSTRAINTS_FILE # activate the virtualenv so that any tools run by dib run # using the python inside it set +u source ./env/bin/activate set -u ./openstack/diskimage-builder/tests/run_functests.sh \ gentoo/build-succeeds \ debian-minimal/stable-vm \ opensuse-minimal/opensuse423-build-succeeds set +u deactivate set -u executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python3-image/000077500000000000000000000000001324273057700266405ustar00rootroot00000000000000diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python3-image/post.yaml000066400000000000000000000006331324273057700305130ustar00rootroot00000000000000- hosts: primary tasks: - name: Copy files from {{ ansible_user_dir }}/workspace/ on node synchronize: src: '{{ ansible_user_dir }}/workspace/' dest: '{{ zuul.executor.log_root }}' mode: pull copy_links: true verify_host: true rsync_opts: - --include=/logs/** - --include=*/ - --exclude=* - --prune-empty-dirs diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python3-image/run.yaml000066400000000000000000000053621324273057700303360ustar00rootroot00000000000000- hosts: all name: Autoconverted job legacy-dib-dsvm-functests-python3-image from old job gate-dib-dsvm-functests-python3-ubuntu-xenial-image-nv tasks: - name: Ensure legacy workspace directory file: path: '{{ ansible_user_dir }}/workspace' state: directory - shell: cmd: | set -u set -e set -x cd ~ /usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git \ git://git.openstack.org \ openstack/diskimage-builder \ openstack/requirements executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -e set -x cd ~/openstack/diskimage-builder /usr/local/jenkins/slave_scripts/install-distro-packages.sh executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -u set -e set -x # # We do things that might exceed the default 2GiB tmpfs, and # use the larger mounted space. cd ~ sudo mkdir /opt/dib_cache sudo chown zuul:zuul /opt/dib_cache export DIB_NO_TMPFS=1 export TMPDIR=/opt/dib_cache virtualenv -p python3 env export UPPER_CONSTRAINTS_FILE=$(pwd)/openstack/requirements/upper-constraints.txt sed -i '/^diskimage-builder/d' $UPPER_CONSTRAINTS_FILE ./env/bin/pip install $(pwd)/openstack/diskimage-builder -c $UPPER_CONSTRAINTS_FILE # TODO(pabelanger): Remove once we migrated to bindep ./openstack/diskimage-builder/tests/install_test_deps.sh # This sets up some repo files pointing to the infra mirrors # which are used during test runs ./openstack/diskimage-builder/contrib/setup-gate-mirrors.sh # UPPER_CONSTRAINTS_FILE is causing side effects when doing image # build, unset it unset UPPER_CONSTRAINTS_FILE # activate the virtualenv so that any tools run by dib run # using the python inside it set +u source ./env/bin/activate set -u ./openstack/diskimage-builder/tests/run_functests.sh \ centos7/build-succeeds \ opensuse/build-succeeds \ opensuse/opensuse423-build-succeeds \ fedora/build-succeeds \ ubuntu/trusty-build-succeeds \ ubuntu/xenial-build-succeeds set +u deactivate set -u executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python3-opensuse-423/000077500000000000000000000000001324273057700277255ustar00rootroot00000000000000diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python3-opensuse-423/post.yaml000066400000000000000000000006331324273057700316000ustar00rootroot00000000000000- hosts: primary tasks: - name: Copy files from {{ ansible_user_dir }}/workspace/ on node synchronize: src: '{{ ansible_user_dir }}/workspace/' dest: '{{ zuul.executor.log_root }}' mode: pull copy_links: true verify_host: true rsync_opts: - --include=/logs/** - --include=*/ - --exclude=* - --prune-empty-dirs diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python3-opensuse-423/run.yaml000066400000000000000000000050001324273057700314100ustar00rootroot00000000000000- hosts: all name: Autoconverted job legacy-dib-dsvm-functests-python3-opensuse-423 from old job gate-dib-dsvm-functests-python3-opensuse-423-nv tasks: - name: Ensure legacy workspace directory file: path: '{{ ansible_user_dir }}/workspace' state: directory - shell: cmd: | set -u set -e set -x cd ~ /usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git \ git://git.openstack.org \ openstack/diskimage-builder \ openstack/requirements executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -e set -x cd ~/openstack/diskimage-builder /usr/local/jenkins/slave_scripts/install-distro-packages.sh executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -u set -e set -x # # We do things that might exceed the default 2GiB tmpfs, and # use the larger mounted space. cd ~ sudo mkdir /opt/dib_cache sudo chown zuul:zuul /opt/dib_cache export DIB_NO_TMPFS=1 export TMPDIR=/opt/dib_cache virtualenv -p python2 env export UPPER_CONSTRAINTS_FILE=$(pwd)/openstack/requirements/upper-constraints.txt sed -i '/^diskimage-builder/d' $UPPER_CONSTRAINTS_FILE ./env/bin/pip install $(pwd)/openstack/diskimage-builder -c $UPPER_CONSTRAINTS_FILE # TODO(pabelanger): Remove once we migrated to bindep ./openstack/diskimage-builder/tests/install_test_deps.sh # This sets up some repo files pointing to the infra mirrors # which are used during test runs ./openstack/diskimage-builder/contrib/setup-gate-mirrors.sh # UPPER_CONSTRAINTS_FILE is causing side effects when doing image # build, unset it unset UPPER_CONSTRAINTS_FILE # activate the virtualenv so that any tools run by dib run # using the python inside it set +u source ./env/bin/activate set -u ./openstack/diskimage-builder/tests/run_functests.sh set +u deactivate set -u executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python3/000077500000000000000000000000001324273057700255605ustar00rootroot00000000000000diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python3/post.yaml000066400000000000000000000006331324273057700274330ustar00rootroot00000000000000- hosts: primary tasks: - name: Copy files from {{ ansible_user_dir }}/workspace/ on node synchronize: src: '{{ ansible_user_dir }}/workspace/' dest: '{{ zuul.executor.log_root }}' mode: pull copy_links: true verify_host: true rsync_opts: - --include=/logs/** - --include=*/ - --exclude=* - --prune-empty-dirs diskimage-builder-2.11.0/playbooks/legacy/dib-dsvm-functests-python3/run.yaml000066400000000000000000000047551324273057700272630ustar00rootroot00000000000000- hosts: all name: Autoconverted job legacy-dib-dsvm-functests-python3 from old job gate-dib-dsvm-functests-python3-ubuntu-xenial tasks: - name: Ensure legacy workspace directory file: path: '{{ ansible_user_dir }}/workspace' state: directory - shell: cmd: | set -u set -e set -x cd ~ /usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git \ git://git.openstack.org \ openstack/diskimage-builder \ openstack/requirements executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -e set -x cd ~/openstack/diskimage-builder /usr/local/jenkins/slave_scripts/install-distro-packages.sh executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' - shell: cmd: | set -u set -e set -x # # We do things that might exceed the default 2GiB tmpfs, and # use the larger mounted space. cd ~ sudo mkdir /opt/dib_cache sudo chown zuul:zuul /opt/dib_cache export DIB_NO_TMPFS=1 export TMPDIR=/opt/dib_cache virtualenv -p python2 env export UPPER_CONSTRAINTS_FILE=$(pwd)/openstack/requirements/upper-constraints.txt sed -i '/^diskimage-builder/d' $UPPER_CONSTRAINTS_FILE ./env/bin/pip install $(pwd)/openstack/diskimage-builder -c $UPPER_CONSTRAINTS_FILE # TODO(pabelanger): Remove once we migrated to bindep ./openstack/diskimage-builder/tests/install_test_deps.sh # This sets up some repo files pointing to the infra mirrors # which are used during test runs ./openstack/diskimage-builder/contrib/setup-gate-mirrors.sh # UPPER_CONSTRAINTS_FILE is causing side effects when doing image # build, unset it unset UPPER_CONSTRAINTS_FILE # activate the virtualenv so that any tools run by dib run # using the python inside it set +u source ./env/bin/activate set -u ./openstack/diskimage-builder/tests/run_functests.sh set +u deactivate set -u executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace' environment: '{{ zuul | zuul_legacy_vars }}' diskimage-builder-2.11.0/pylint.cfg000066400000000000000000000001401324273057700171610ustar00rootroot00000000000000[MASTER] persistent=no [MESSAGES CONTROL] disable=all enable=bad-indentation, logging-not-lazy diskimage-builder-2.11.0/releasenotes/000077500000000000000000000000001324273057700176575ustar00rootroot00000000000000diskimage-builder-2.11.0/releasenotes/notes/000077500000000000000000000000001324273057700210075ustar00rootroot00000000000000diskimage-builder-2.11.0/releasenotes/notes/1.16.0-updates-bad91fc0b36c1755.yaml000066400000000000000000000013041324273057700263460ustar00rootroot00000000000000--- fixes: - The ``yum-minimal`` element now correctly strips locale-archive files, but leaves en.US/en.US-UTF8 for compatibility with remote access. Better failure detection has been added to avoid leaving lock files behind if initial installs to the chroot fail. - The ``epel`` element was fixed to not incorrectly install EPEL6 on Centos 7. EPEL is added as a requirement of ``centos-minimal`` (previously, it was tacitly required but not expressed). - The ``dib-lint`` tool is now more verbose, has better documentation and more correctly detects errors in JSON & YAML. - The ``ironic-agent`` element no longer removes the ``sudo`` package in its cleanup phase. diskimage-builder-2.11.0/releasenotes/notes/1.17.0-ef744f36d277dba4.yaml000066400000000000000000000022231324273057700247230ustar00rootroot00000000000000--- features: - The ``dib-lint`` tool now has a best-effort sudo check to help flag uses of ``sudo`` outside the chroot environment. This can help avoid some classes of breakout problems that can interfere with the host system. - Running with ``-x`` will give a more verbose log output that includes function and line-numbers. Work is ongoing to further reduce noise in the verbose logs to make it as useful as possible. Feedback is welcome on the new format. - the ``pkg-map`` element can now install different packages based on the distribution release. fixes: - Locale stripping for ``yum-minimal`` has been reworked to be more reliable for Fedora 23 and CentOS. - The Ubuntu Precise kernel is too old to use the online ``growroot`` element, so we returned to using ``cloud-initramfs-growroot`` in the ``debootstrap`` element for it. - The ``dib-lint`` tool now better matches python files. - The ``apt-sources`` element was fixed to work with the ``debian-minimal`` element. - Dependencies for the systemd target in ``dhcp-all-interfaces`` were updated to correctly wait for interfaces. diskimage-builder-2.11.0/releasenotes/notes/1.18.0-4433d3076627f10d.yaml000066400000000000000000000003471324273057700244150ustar00rootroot00000000000000--- features: - The ``fedora-minimal`` element now supports Fedora 24 - The ``pkg-map`` element now correctly supports images without python2 in the default install (such as the latest releases from Ubuntu and Fedora). diskimage-builder-2.11.0/releasenotes/notes/1.18.1-ceeb514708dcb731.yaml000066400000000000000000000004211324273057700247070ustar00rootroot00000000000000--- fixes: - Fix issue where ironic-agent did not work on ppc due to missing dmidecode. - Fix issue where we were unable to create empty tempdirs in the target image. - Revert change where we were installing virtualenv on centos with both system and python packages. diskimage-builder-2.11.0/releasenotes/notes/block-device-handling-279cddba8a859718.yaml000066400000000000000000000017461324273057700302500ustar00rootroot00000000000000--- features: - Add new block device handling. Unify and generalize the creation and usage of block device. This release includes setting up the general infrastructure and setting up loop device issues: - Because the implementation of the new block device layer is not complete, some features which are already implemented cannot be used because of limitations of the current environment. deprecations: - The current way of handling block devices is deprecated. The existing block device phase will be called for a limited time. If this phase delivers a result, this is used; if there is no result, the new way of block device handling is used. Please note that the old way of block device handling has some major limitations such as that it is only possible to use one block device. - The element 'partitioning-sfdisk' is deprecated. The new implementation will create the partition tables based on a tool independent description. diskimage-builder-2.11.0/releasenotes/notes/block-device-lvm-c3b8a214952b4db5.yaml000066400000000000000000000001011324273057700272140ustar00rootroot00000000000000--- features: - Adds lvm support, allowing to create volumes. diskimage-builder-2.11.0/releasenotes/notes/block-device-mkfs-mount-fstab-42d7efe28fc2df04.yaml000066400000000000000000000001161324273057700317740ustar00rootroot00000000000000--- features: - | Adds mkfs, mount and fstab to the block device layer. diskimage-builder-2.11.0/releasenotes/notes/block-device-partitioning-237249e7ed2bad26.yaml000066400000000000000000000005531324273057700311500ustar00rootroot00000000000000--- features: - Create partitions with MBR layout optimized for performance and highly configurable. deprecations: - The new partitions are created based on configuration rather than on a list of provided commands for a special partitioning tool. Therefore elements using tools (like partitioning-sfdisk) are deprecated and will be removed. diskimage-builder-2.11.0/releasenotes/notes/centos-retired-f17ae9f6f03e57e3.yaml000066400000000000000000000014121324273057700271300ustar00rootroot00000000000000--- deprecations: - | The ``centos`` and ``rhel`` elements have been removed. These were building version 6 which is no longer supported (mostly due to a lack of python 2.7). Version 7 support is available via the ``centos7`` and ``rhel7`` elements (which downloads and modifies the upstream cloud images) or via ``centos-minimal`` (which builds the image from a empty chroot; only available for CentOS). ``centos-minimal`` is suggested as this is what OpenStack Infrastructure uses for its elements. Unfortunately, ``centos-minimal`` sets ``DISTRO=centos`` while ``centos7`` sets ``DISTRO=centos7``, despite building the same thing as far as upper levels are concerned. We plan to rectify this in the version 8 time-frame. diskimage-builder-2.11.0/releasenotes/notes/dash-p-after-install-58a87549c1c906c3.yaml000066400000000000000000000004061324273057700277020ustar00rootroot00000000000000other: - The packages specified with the `-p` command-line operation are now installed after the `install.d` phase, not before. This is to give elements priority when installing packages. The flag documentation has been updated to describe this. diskimage-builder-2.11.0/releasenotes/notes/dib-distribution-mirror-8c241c0d3d4a539a.yaml000066400000000000000000000006331324273057700306640ustar00rootroot00000000000000--- deprecations: - The ``DIB_[DISTRO]_DISTRIBUTION_MIRROR`` variables have been removed. These were undocumented ways to set ``DIB_DISTRIBUTION_MIRROR`` for some elements. It was not implemented consistently and causing some confusion. If you need to setup mirrors for testing purposes, the ``openstack-ci-mirrors`` element is a good example that is used by OpenStack gate testing.dib-init-system_fix_for_debian_jessie-c6f7261ee84dad27.yaml000066400000000000000000000003211324273057700335150ustar00rootroot00000000000000diskimage-builder-2.11.0/releasenotes/notes--- fixes: - dib-init-system did not correctly find the init system for Debian Jessie and Debian Stretch. This version also looks for /bin/systemctl as as hint for systemd and fixes the problem. diskimage-builder-2.11.0/releasenotes/notes/dib-run-parts-6f67d038aa5a4156.yaml000066400000000000000000000007271324273057700265250ustar00rootroot00000000000000--- deprecations: - | The ``dib-run-parts`` element is no longer required as ``disk-image-create`` will directly source the internal version for running scripts within the chroot. This element was unintentionally leaving ``/usr/local/bin/dib-run-parts`` in the built image. From code search we do not believe anyone was relying on the presence of this script. If you do require it, you should source the ``dib-utils`` package to install. diskimage-builder-2.11.0/releasenotes/notes/dib-run-parts-e18cc3a6c2d66c24.yaml000066400000000000000000000006151324273057700266550ustar00rootroot00000000000000--- deprecations: - | dib no longer exports ``dib-run-parts``. Adding this was an oversight made during v2 development, since ``dib-utils`` already provides this. The ``dib-run-parts`` used internally (``diskimage_builder/lib/dib-run-parts``) is not intended to be used by external tools. If you require ``dib-run-parts``, you should install the ``dib-utils`` package.diskimage-builder-2.11.0/releasenotes/notes/dibv2-omnibus-b30e0c7ecd76db8d.yaml000066400000000000000000000101251324273057700270060ustar00rootroot00000000000000--- prelude: >- Version 2.0.0 of diskimage-builder incorporates recent work from the feature/v2 branch. This includes incorporating some largely internal changes to the way it finds and calls elements, enhancements to partitioning and removal of some long-deprecated elements. If you use dib exclusively via the command-line disk-image-create installed from a package or via pypi you are unlikely to notice any difference (if you run it directly from a git-tree checkout, you may be affected). features: - | 2.0.0 includes a new framework for partitioning contributed by Andreas Florath. This should allow for creating multiple partitions, images with encryption, LVM support and flexibility for multiple-devices, all of which are currently not supported. Please check the v2 documentation, specs and reach out if these features interest you (some parts still in review). - | Element override is now supported. If you have an element of the same name earlier in the ``ELEMENTS_PATH``, it will override later instances (previously, the behaviour was undefined). upgrade: - | For purposes of both users and development we want dib to be as "pythonic" as possible and behave like all other projects. Two major visible changes are: - command-line scripts are entry points (i.e. need to be installed) - elements have moved under diskimage_create module The result of the first is that ``./bin/disk-image-create`` from the source tree is no longer there. Like all other projects, you should install dib into a virtualenv (if you're developing, use pip -e) and ``disk-image-create`` will "just work". The second change, moving the inbuilt elements under the ``diskimage_create`` module, is a simplification so we always have a canonical path to our elements. Since we now always know where elements are relative to the imported diskimage_builder module we can drop all the path guessing complexity. This has other good flow-on effects such as ``testr`` being able to find unit-tests for elements in the normal fashion and having imports work as usual. We are aware there are a number of tools that like to take dib elements and do things with them. Reading some of the dib source you may find there is a canonical way to find out the included dib elements path -- ask dib itself, something like .. code-block:: shell DIB_ELEMENTS=$(python -c ' import diskimage_builder.paths; diskimage_builder.paths.show_path("elements")') Note you probably do not want this. As mentioned, another feature of v2 is override elements -- an element that appears first in the element path-list will override any built-in one (just like $PATH). There is a function, ``diskimage_builder.get_elements()``, which will correctly process the element path, calculate overrides and return a canonical list of elements, their dependencies and correct paths. *That* said, you probably do not want this either! There are a number of elements that do things on behalf of other elements -- they look for a file in the included elements, say, and use that as a manifest for something. Previously, these would just have to make up their own element processing via inspection of the command-line arguments. dib now exports pre-computed variables that an element can walk for all the current build elements -- a YAML list for easy python decoding and a function that builds an array for Bash elements. deprecations: - | A number of long-deprecated elements have been removed in v2, which are to the best of our knowledge unused. - ``partitioning-sfdisk`` - ``deploy-ironic-element`` - ``ironc-discovered-ramdisk`` - ``serial-console-element`` - ``map-services`` - | We have removed and deprecated the ``dib-utils`` package. This was intended to be a more generic repository of tools that might be useful outside dib, but that did not eventuate and it has been folded back into dib for simplicity. diskimage-builder-2.11.0/releasenotes/notes/doc-auto-element-dependency-cb7488c5bb7301a4.yaml000066400000000000000000000002051324273057700313570ustar00rootroot00000000000000--- features: - Create sphinx directive 'element_deps' that automatically generates dependencies in the element documentation. diskimage-builder-2.11.0/releasenotes/notes/dracut-network-adaabf90da9f6866.yaml000066400000000000000000000002161324273057700273060ustar00rootroot00000000000000--- deprecations: - The ``dracut-network`` element has been removed. It has not been functioning for some time due to incorrect paths. diskimage-builder-2.11.0/releasenotes/notes/element-info-entry-point-448bf622be6061a0.yaml000066400000000000000000000005761324273057700306770ustar00rootroot00000000000000--- deprecations: - The ``element-info`` script is now provided by a standard python entry-point, rather than an explicit wrapper script. This may affect you if you were running this script directly out of ``bin`` in the source directory without installing. See developer notes for details on using developer-installs with virtual environments for testing. diskimage-builder-2.11.0/releasenotes/notes/element-override-ccda78c24ab4a4ff.yaml000066400000000000000000000003561324273057700276550ustar00rootroot00000000000000--- deprecations: - Element override behavior is now defined, with elements found in earlier entries of ``ELEMENTS_PATH`` overriding later ones (e.g. the same semantics as ``$PATH``). Previously the behavior was undefined.diskimage-builder-2.11.0/releasenotes/notes/element-vars-c6bf2e6795002f01.yaml000066400000000000000000000003331324273057700264250ustar00rootroot00000000000000--- features: - Elements that need access to the other elements being used during the build should use the new ``IMAGE_ELEMENT_YAML`` environment variable and it's Bash equivalent ``get_image_element_array``.diskimage-builder-2.11.0/releasenotes/notes/fedora26-690b9fd9ac3c3d4f.yaml000066400000000000000000000003011324273057700256620ustar00rootroot00000000000000--- upgrade: - The ``fedora-minimal`` and ``fedora`` elements have been updated to default to Fedora 26. Note if you to pin to specific versions, be sure to specify ``DIB_RELEASE``. diskimage-builder-2.11.0/releasenotes/notes/grub-timeout-1cdd14a2b1467d89.yaml000066400000000000000000000003621324273057700265310ustar00rootroot00000000000000--- features: - The ``bootloader`` element will explicitly set the timeout to ``5`` seconds when using ``grub`` (previously this was undefined, but platform defaults were usually 5 seconds). Set this to ``0`` for faster boots. diskimage-builder-2.11.0/releasenotes/notes/incorrect-grub-label-5d2000215c0cc73e.yaml000066400000000000000000000002301324273057700300000ustar00rootroot00000000000000--- fixes: - | This fixes bug 1742170 where the grub root label is different than the file system label when booting from a whole disk image. diskimage-builder-2.11.0/releasenotes/notes/move_tidy_logs_to_main-a8c03427fe1a445c.yaml000066400000000000000000000006061324273057700306340ustar00rootroot00000000000000--- features: - Cleaning logs was split, some was done in the img-functions.finalise_base, some was done in the base element. The version unifies tidy up logs in the lib/img-functions. Especially when building docker container images the base element cannot be used. This patch removes about some hundreds KB of useless logs in cases when the base element is not used.diskimage-builder-2.11.0/releasenotes/notes/openssh-server-0f6d065748a2fc18.yaml000066400000000000000000000001711324273057700270150ustar00rootroot00000000000000--- features: - New openssh-server element to ensure that the openssh server is installed and enabled during boot. diskimage-builder-2.11.0/releasenotes/notes/opensuse-423-default-3bc73fff69374cd0.yaml000066400000000000000000000003631324273057700277730ustar00rootroot00000000000000--- upgrade: - | The opensuse and opensuse-minimal element are now defaulting to 42.3, which is the latest stable openSUSE release. Building for 42.2 is still supported, however requires setting DIB_RELEASE to '42.2' explicitly. diskimage-builder-2.11.0/releasenotes/notes/opensuse-minimal-45267f5be1112c22.yaml000066400000000000000000000004331324273057700272230ustar00rootroot00000000000000--- features: - New zypper-minimal and opensuse-minimal elements to create basic openSUSE images. These two new elements are also making use of the existing zypper element which has been extended to include the functionality previously present in the opensuse element. diskimage-builder-2.11.0/releasenotes/notes/package-install-arch-38bb5a6e61794fa5.yaml000066400000000000000000000004341324273057700300720ustar00rootroot00000000000000--- features: - The ``package-installs`` element now supports a list of architectures to install the package for in the ``arch`` field and negative matching with a ``not-arch`` field. This allows greater flexibility when installing packages for only some architectures.diskimage-builder-2.11.0/releasenotes/notes/package-outside-debootstrap-ac93e9ce991819f1.yaml000066400000000000000000000002651324273057700315260ustar00rootroot00000000000000--- fixes: - The `debian-minimal` and and `ubuntu-minimal` elements now install directly from the updates repo, avoiding the need to double-install packages during build. diskimage-builder-2.11.0/releasenotes/notes/remove-dib-utils-37f70dfad54900a0.yaml000066400000000000000000000003171324273057700272710ustar00rootroot00000000000000--- deprecations: - The `dib-utils` requirement has been removed as the `dib-run-parts` script is now shipped from within diskimage-builder. The `dib-utils` project is now considered retired. diskimage-builder-2.11.0/releasenotes/notes/runtime-ssh-host-keys-7a2fc873cc90d33e.yaml000066400000000000000000000004531324273057700303770ustar00rootroot00000000000000--- features: - New element (runtime-ssh-host-keys) to manage SSH host keys at boot. Since SSH host key generation is not standard across operating systems, add support for both Debian and Ubuntu to handle it. While this is a new element, simple-init has been updated to depend on it. diskimage-builder-2.11.0/releasenotes/notes/squashfs-output-91c1f0dc37474d3c.yaml000066400000000000000000000000641324273057700273010ustar00rootroot00000000000000--- features: - New squashfs image output format. diskimage-builder-2.11.0/releasenotes/notes/start-using-reno-446b3d52a467a273.yaml000066400000000000000000000011431324273057700271650ustar00rootroot00000000000000--- fixes: - We now use ``fstrim`` rather than ``zerofree`` to reduce final image size. This is considered current best practice. - Fix an occasional failure to build ``centos-minimal`` and ``fedora-minimal`` due to suspected rpmdb failures. The main symptom was ``dnf`` or ``yum`` thinking packages were not installed when they were. - The ``debian-minimal`` element now obeys ``DIB_DEBIAN_COMPONENTS`` for adding components. - Fix some issues with Gentoo, and add to supported distributions. Testing is currently non-voting other: - Start using reno for release notes diskimage-builder-2.11.0/releasenotes/notes/sysprep-f3fd036bc1d2c405.yaml000066400000000000000000000004061324273057700256610ustar00rootroot00000000000000--- features: - | Adds sysprep element included by all systemd distros fixes: - | Adds default sysprep element clearing /etc/machine-id which prevents duplicated /etc/machine-id by forcing systemd to generate a new id for each booted system. diskimage-builder-2.11.0/releasenotes/notes/timestamp-43015aa5434e8ddb.yaml000066400000000000000000000012251324273057700261010ustar00rootroot00000000000000--- features: - A ``--logfile`` option is added to save output to a given file. - By default, all ``stdout`` will log with timestamps (this used to be inconsistent; python tools logged with a timestamp, and bash parts did not). If you set ``DIB_NO_TIMESTAMP`` to ``1`` the timestamp prefix will be suppressed (this would be appropriate if you are running diskimage-builder and capturing its output, and adding your own timestamp). Note that output to ``--logfile`` will always be timestamped. - You can set ``DIB_QUIET`` to ``1`` to suppress all output. This is likely only useful when used with the ``--logfile`` command.diskimage-builder-2.11.0/releasenotes/notes/upgrade-pip-before-c-d2443847f9d58c7a.yaml000066400000000000000000000005301324273057700277420ustar00rootroot00000000000000--- fixes: - | This fixes bug 1744403 where 60-ironic-agent-install fails to run when DIB_INSTALLTYPE_ironic_agent=source is set. pip installs constraints with the -c argument and this argument is a relatively newer addion. The currently installed pip may not support that option and must be upgraded before proceeding. diskimage-builder-2.11.0/releasenotes/notes/yum-cache-removal-148c33012515e56e.yaml000066400000000000000000000007431324273057700272030ustar00rootroot00000000000000--- deprecations: - The ``DIB_YUMCHROOT_USE_CACHE`` variable has been removed and the Fedora and CentOS ``-minimal`` initial chroot will always be created by the package manager. The default creation of a chroot tarball is stopped for these elements. This unused option was unsafe; there is no guarantee that the base system will not change even between runs. Getting the package manager to reuse the cache for the initial chroot install is future work. diskimage-builder-2.11.0/releasenotes/source/000077500000000000000000000000001324273057700211575ustar00rootroot00000000000000diskimage-builder-2.11.0/releasenotes/source/conf.py000066400000000000000000000166471324273057700224740ustar00rootroot00000000000000import sys import os # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = ['openstackdocstheme', 'reno.sphinxext'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'diskimage-builder' copyright = u'2016, Diskimage-builder contributors' # openstackdocstheme options repository_name = 'openstack/diskimage-builder' bug_project = 'diskimage-builder' bug_tag = '' html_last_updated_fmt = '%Y-%m-%d %H:%M' # Release notes do not need a version in the title, they span # multiple versions. # The short X.Y version. release = '' # The full version, including alpha/beta/rc tags. version = '' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = [] # The reST default role (used for this markup: `text`) to use for all # documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. #keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'openstackdocs' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". #html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. #html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'diskimage-builderdoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ ('index', 'diskimage-builder-releasenotes.tex', u'diskimage-builder Documentation', u'OpenStack', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'diskimage-builder', u'diskimage-builder Release Notes', [u'OpenStack'], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'diskimage-builder', u'diskimage-builder Release Notes', u'OpenStack', 'diskimage-builder', 'Build disk images for CI and more.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False # -- Options for Internationalization output ------------------------------ locale_dirs = ['locale/'] diskimage-builder-2.11.0/releasenotes/source/index.rst000066400000000000000000000001641324273057700230210ustar00rootroot00000000000000=============================== diskimage-builder Release Notes =============================== .. release-notes:: diskimage-builder-2.11.0/requirements.txt000066400000000000000000000006061324273057700204540ustar00rootroot00000000000000# The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. Babel!=2.4.0,>=2.3.4 # BSD networkx<2.0,>=1.10 # BSD pbr!=2.1.0,>=2.0.0 # Apache-2.0 PyYAML>=3.10 # MIT flake8<2.6.0,>=2.5.4 # MIT six>=1.10.0 # MIT stevedore>=1.20.0 # Apache-2.0 diskimage-builder-2.11.0/setup.cfg000066400000000000000000000047051324273057700170150ustar00rootroot00000000000000[metadata] name = diskimage-builder summary = Golden Disk Image builder. description-file = README.rst author = OpenStack author-email = openstack-dev@lists.openstack.org license: Apache License (2.0) home-page = https://docs.openstack.org/diskimage-builder/latest/ classifier = Development Status :: 5 - Production/Stable License :: OSI Approved :: Apache Software License Operating System :: POSIX :: Linux Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 Environment :: Console [build_sphinx] all_files = 1 build-dir = doc/build source-dir = doc/source warning-is-error = 1 [pbr] autodoc_index_modules = 1 autodoc_exclude_modules = setup.py diskimage_builder.tests.* diskimage_builder.block_device.tests.* # elements are a weird mix of scripts & python diskimage_builder.elements.* # this is non-pythony stuff diskimage_builder.lib.* [files] packages = diskimage_builder # ramdisk-image-create is a symlink which becomes a # real file on install; see lp bug #1212482 scripts = bin/dib-lint data_files = share/diskimage-builder/scripts = scripts/* [extract_messages] keywords = _ gettext ngettext l_ lazy_gettext mapping_file = babel.cfg output_file = diskimage_builder/locale/diskimage_builder.pot [update_catalog] domain = diskimage_builder output_dir = diskimage_builder/locale input_file = diskimage_builder/locale/diskimage_builder.pot [compile_catalog] directory = diskimage_builder/locale domain = diskimage_builder [wheel] universal = 1 [entry_points] console_scripts = element-info = diskimage_builder.element_dependencies:main dib-block-device = diskimage_builder.block_device.cmd:main disk-image-create = diskimage_builder.disk_image_create:main ramdisk-image-create = diskimage_builder.disk_image_create:main diskimage_builder.block_device.plugin = local_loop = diskimage_builder.block_device.level0.localloop:LocalLoop partitioning = diskimage_builder.block_device.level1.partitioning:Partitioning lvm = diskimage_builder.block_device.level1.lvm:LVMPlugin mkfs = diskimage_builder.block_device.level2.mkfs:Mkfs mount = diskimage_builder.block_device.level3.mount:Mount fstab = diskimage_builder.block_device.level4.fstab:Fstab # unit test extensions diskimage_builder.block_device.plugin_test = test_a = diskimage_builder.block_device.tests.plugin.test_a:TestA test_b = diskimage_builder.block_device.tests.plugin.test_b:TestB diskimage-builder-2.11.0/setup.py000066400000000000000000000020061324273057700166760ustar00rootroot00000000000000# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. # THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT import setuptools # In python < 2.7.4, a lazy loading of package `pbr` will break # setuptools if some other modules registered functions in `atexit`. # solution from: http://bugs.python.org/issue15881#msg170215 try: import multiprocessing # noqa except ImportError: pass setuptools.setup( setup_requires=['pbr>=2.0.0'], pbr=True) diskimage-builder-2.11.0/test-requirements.txt000066400000000000000000000010611324273057700214250ustar00rootroot00000000000000# The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. hacking<0.11,>=0.10.0 pylint==1.7.1 # GPLv2 fixtures>=3.0.0 # Apache-2.0/BSD mock>=2.0.0 # BSD oslotest>=3.2.0 # Apache-2.0 testrepository>=0.0.18 # Apache-2.0/BSD testtools>=2.2.0 # MIT # Doc requirements sphinx!=1.6.6,>=1.6.2 # BSD openstackdocstheme>=1.18.1 # Apache-2.0 # releasenotes reno>=2.5.0 # Apache-2.0 coverage!=4.4,>=4.0 # Apache-2.0 diskimage-builder-2.11.0/tests/000077500000000000000000000000001324273057700163305ustar00rootroot00000000000000diskimage-builder-2.11.0/tests/README.rst000066400000000000000000000005241324273057700200200ustar00rootroot00000000000000DIB Testing =========== There are several interfaces for testing various parts of DIB * run_dib_library_tests.sh : run unit-tests for internal diskimage-builder library functions * run_functests.sh : run functional testing provided by elements * run_output_format_test.sh : run a test of image conversion (requires docker) diskimage-builder-2.11.0/tests/elements/000077500000000000000000000000001324273057700201445ustar00rootroot00000000000000diskimage-builder-2.11.0/tests/elements/fake-os/000077500000000000000000000000001324273057700214715ustar00rootroot00000000000000diskimage-builder-2.11.0/tests/elements/fake-os/README.rst000066400000000000000000000002401324273057700231540ustar00rootroot00000000000000fake-os ======= This is not the element you are looking for. This element does not provide an operating system, it provides a fake for testing purposes only. diskimage-builder-2.11.0/tests/elements/fake-os/element-provides000066400000000000000000000000211324273057700246670ustar00rootroot00000000000000operating-system diskimage-builder-2.11.0/tests/elements/fake-os/root.d/000077500000000000000000000000001324273057700226765ustar00rootroot00000000000000diskimage-builder-2.11.0/tests/elements/fake-os/root.d/10-fake-os000077500000000000000000000005321324273057700243670ustar00rootroot00000000000000#!/bin/bash set -eux set -o pipefail sudo touch $TARGET_ROOT/fake-os for DIR in etc mnt proc dev sys tmp; do sudo mkdir -p ${TARGET_ROOT}/${DIR} done # We need some size so the mkfs does not fail when making an fs of size 0 # We also need to account for the journal size on the FS dd if=/dev/zero of=$TARGET_ROOT/fake-data bs=1M count=42 diskimage-builder-2.11.0/tests/install_test_deps.sh000077500000000000000000000016751324273057700224200ustar00rootroot00000000000000#!/bin/bash set -eux set -o pipefail sudo apt-get update || true sudo apt-get install -y --force-yes \ apt-transport-https \ bzip2 \ debootstrap \ docker.io \ inetutils-ping \ lsb-release \ kpartx \ python-lzma \ qemu-utils \ rpm \ uuid-runtime \ yum-utils || \ sudo yum -y install --enablerepo=epel \ bzip2 \ dpkg \ debootstrap \ docker \ kpartx \ util-linux \ qemu-img \ policycoreutils-python || \ sudo zypper -n install \ bzip2 \ debootstrap \ docker \ kpartx \ util-linux \ python-pyliblzma \ yum-utils \ qemu-tools || \ sudo emerge \ app-arch/bzip2 \ app-emulation/qemu \ dev-python/pyyaml \ sys-block/parted \ sys-fs/multipath-tools \ qemu-img \ yum-utils diskimage-builder-2.11.0/tests/run_dib_library_tests.sh000077500000000000000000000025751324273057700232700ustar00rootroot00000000000000#!/bin/bash # unit testing for some of the common-functions # # This is fairly invasive and *may* leave behind mounts, etc, that # need a human in the loop. Thus it's mostly useful for developers # during testing, but not so great for CI source ../diskimage_builder/lib/common-functions # # Directory mounting and unmounting # # make mount points TMP_DIR=$(mktemp -d) cd $TMP_DIR mkdir mnt mkdir mnt/proc mnt/dev mnt/dev/pts mnt/sysfs mnt/sys # for extra complexity, simulate the path being behind a symlink ln -s mnt mnt-symlink TMP_MOUNT_PATH=$TMP_DIR/mnt-symlink # mount devices mount_proc_dev_sys if [ $(grep "$TMP_DIR" /proc/mounts | wc -l) -ne 4 ]; then echo "*** FAILED to mount all directories" # we might be in an unclean state here, but something is broken... # we don't want to delete mounted system directories return 1 else echo "*** PASS : mounted all directories" fi # umount devices unmount_dir $TMP_MOUNT_PATH if grep -q "$TMP_DIR" /proc/mounts; then echo "*** FAILED due to mounts being left behind" return 1 else echo "*** PASS all directories unmounted" fi # unmount missing dir if unmount_dir /this/path/does/not/exist/but/this/should/not/fail; then echo "*** PASS unmount_dir ignored a missing path" else echo "*** FAILED unmount_dir should ignore missing paths" return 1 fi # cleanup rm -rf $TMP_DIR ### TODO more tests here diskimage-builder-2.11.0/tests/run_functests.sh000077500000000000000000000256261324273057700216040ustar00rootroot00000000000000#!/bin/bash set -eu set -o pipefail BASE_DIR=$(cd $(dirname "$0")/.. && pwd) # then execute tests for elements export DIB_CMD=disk-image-create export DIB_ELEMENTS=$(python -c ' import diskimage_builder.paths diskimage_builder.paths.show_path("elements")') # Setup sane locale defaults, because this information is leaked into DIB. export LANG=en_US.utf8 export LC_ALL= # # Default skip tests # # For time reasons, we do not run these tests by default; i.e. these # tests are not run by "tox -e func" in the gate. # DEFAULT_SKIP_TESTS=( ## These are part of the "extras-nv" job # These require "zypper" on the host which is not available on # all platforms opensuse-minimal/build-succeeds opensuse-minimal/opensuse423-build-succeeds # non-voting; not used by infra currently gentoo/build-succeeds # Needs infra mirroring to move to voting job debian-minimal/stable-build-succeeds debian-minimal/stable-vm ## # These download base images which has shown to be very unreliable # in the gate. Keep them in a -nv job until we can figure out # better caching for the images opensuse/build-succeeds opensuse/opensuse423-build-succeeds centos7/build-succeeds debian/build-succeeds fedora/build-succeeds ubuntu/trusty-build-succeeds ubuntu/xenial-build-succeeds # No longer reasonable to test upstream (lacks a mirror in infra) # Note this is centos6 and should probably be removed centos/build-succeeds # This job is a bit unreliable, even if we get mirroring debian-minimal/testing-build-succeeds ) # The default output formats (specified to disk-image-create's "-t" # command. Elements can override with a test-output-formats file DEFAULT_OUTPUT_FORMATS="tar" function log_with_prefix { local pr=$1 local log while read a; do log="[$pr] $a" # note: dib logs have timestamp by default now echo "${log}" done } # Log job control messages function log_jc { local msg="$1" local log="[JOB-CONTROL] ${msg}" if [[ ${LOG_DATESTAMP} -ne 0 ]]; then log="$(date +"%Y%m%d-%H%M%S.%N") ${log}" fi echo "${log}" } function job_cnt { running_jobs=$(jobs -p) echo ${running_jobs} | wc -w } # This is needed, because the better 'wait -n' is # available since bash 4.3 only. function wait_minus_n { if [ "${BASH_VERSINFO[0]}" -gt 4 \ -o "${BASH_VERSINFO[0]}" = 4 \ -a "${BASH_VERSINFO[1]}" -ge 3 ]; then # Good way: wait on any job wait -n return $? else # Not that good way: wait on one specific job # (others may be finished in the mean time) local wait_for_pid=$(jobs -p | head -1) wait ${wait_for_pid} return $? fi } # run_disk_element_test # Run a disk-image-build build of ELEMENT including any elements # specified by TEST_ELEMENT. Pass OUTPUT_FORMAT to "-t" function run_disk_element_test() { local test_element=$1 local element=$2 local dont_use_tmp=$3 local output_format="$4" local use_tmp_flag="" local dest_dir=$(mktemp -d) if [[ ${KEEP_OUTPUT} -ne 1 ]]; then trap "rm -rf $dest_dir" EXIT fi if [ "${dont_use_tmp}" = "yes" ]; then use_tmp_flag="--no-tmpfs" fi if break="after-error" break_outside_target=1 \ break_cmd="cp -v \$TMP_MOUNT_PATH/tmp/dib-test-should-fail ${dest_dir} || true" \ DIB_SHOW_IMAGE_USAGE=1 \ ELEMENTS_PATH=$DIB_ELEMENTS/$element/test-elements \ $DIB_CMD -x -t ${output_format} \ ${use_tmp_flag} \ -o $dest_dir/image -n $element $test_element 2>&1 \ | log_with_prefix "${element}/${test_element}"; then if [[ "qcow2" =~ "$output_format" ]]; then if ! [ -f "$dest_dir/image.qcow2" ]; then echo "Error: qcow2 build failed for element: $element, test-element: $test_element." echo "No image $dest_dir/image.qcow2 found!" exit 1 fi fi # check inside the tar for sentinel files if ! [ -f "$dest_dir/image.tar" ]; then echo "Error: Build failed for element: $element, test-element: $test_element." echo "No image $dest_dir/image.tar found!" exit 1 else if tar -tf $dest_dir/image.tar | grep -q /tmp/dib-test-should-fail; then echo "Error: Element: $element, test-element $test_element should have failed, but passed." exit 1 else echo "PASS: Element $element, test-element: $test_element" fi fi else if [ -f "${dest_dir}/dib-test-should-fail" ]; then echo "PASS: Element $element, test-element: $test_element" else echo "Error: Build failed for element: $element, test-element: $test_element." exit 1 fi fi rm -f /tmp/dib-test-should-fail if [[ ${KEEP_OUTPUT} -ne 1 ]]; then # reset trap and cleanup trap EXIT rm -rf $dest_dir fi } # run_ramdisk_element_test # Run a disk-image-builder default build of ELEMENT including any # elements specified by TEST_ELEMENT function run_ramdisk_element_test() { local test_element=$1 local element=$2 local dest_dir=$(mktemp -d) if ELEMENTS_PATH=$DIB_ELEMENTS/$element/test-elements \ $DIB_CMD -x -o $dest_dir/image $element $test_element \ | log_with_prefix "${element}/${test_element}"; then # TODO(dtantsur): test also kernel presence once we sort out its naming # problem (vmlinuz vs kernel) if ! [ -f "$dest_dir/image.initramfs" ]; then echo "Error: Build failed for element: $element, test-element: $test_element." echo "No image $dest_dir/image.initramfs found!" exit 1 else echo "PASS: Element $element, test-element: $test_element" fi else echo "Error: Build failed for element: $element, test-element: $test_element." exit 1 fi } # # run_functests.sh # run the functional tests for dib elements # # find elements that have functional test elements. TESTS will be an # array with each value being "element/test-element" TESTS=() for e in $DIB_ELEMENTS/*/test-elements/*; do test_element=$(echo $e | awk 'BEGIN {FS="/"}{print $NF}') element=$(echo $e | awk 'BEGIN {FS="/"}{print $(NF-2)}') TESTS+=("$element/$test_element") done # # Default values # JOB_MAX_CNT=1 LOG_DATESTAMP=0 KEEP_OUTPUT=0 # # Parse args # while getopts ":hlj:t" opt; do case $opt in h) echo "run_functests.sh [-h] [-l] ..." echo " -h : show this help" echo " -l : list available tests" echo " -j : parallel job count (default to 1)" echo " -t : prefix log messages with timestamp" echo " -k : keep output directories" echo " : functional test to run" echo " Special test 'all' will run all tests" exit 0 ;; l) echo "The available functional tests are:" echo for t in ${TESTS[@]}; do echo -n " $t" if [[ " ${DEFAULT_SKIP_TESTS[@]} " =~ " ${t} " ]]; then echo " [skip]" else echo " [run]" fi done echo exit 0 ;; j) JOB_MAX_CNT=${OPTARG} echo "Running parallel - using [${JOB_MAX_CNT}] jobs" ;; t) LOG_DATESTAMP=1 ;; k) KEEP_OUTPUT=1 ;; \?) echo "Invalid option: -$OPTARG" exit 1 ;; esac done shift $((OPTIND-1)) DONT_USE_TMP="no" if [ "${JOB_MAX_CNT}" -gt 1 ]; then # switch off using tmp dir for image building # (The mem check using the tmp dir is currently done # based on the available memory - and not on the free. # See #1618124 for more details) DONT_USE_TMP="yes" fi # cull the list of tests to run into TESTS_TO_RUN TESTS_TO_RUN=() title="" if [[ -z "$@" ]]; then # remove the skipped tests title="Running default tests:" for test in "${TESTS[@]}"; do if [[ " ${DEFAULT_SKIP_TESTS[@]} " =~ " ${test} " ]]; then continue else TESTS_TO_RUN+=("${test}") fi done elif [[ $1 == "all" ]]; then title="Running all tests:" TESTS_TO_RUN=("${TESTS[@]}") else title="Running specified tests:" for test in $@; do if [[ ! " ${TESTS[@]} " =~ " ${test} " ]]; then echo "${test} : not a known test (see -l)" exit 1 fi TESTS_TO_RUN+=("${test}") done fi # print a little status info echo "------" echo ${title} for test in "${TESTS_TO_RUN[@]}"; do echo " ${test}" done echo "------" function wait_and_exit_on_failure { local pid=$1 wait ${pid} result=$? if [ "${result}" -ne 0 ]; then exit ${result} fi return 0 } EXIT_CODE=0 for test in "${TESTS_TO_RUN[@]}"; do running_jobs_cnt=$(job_cnt) log_jc "Number of running jobs [${running_jobs_cnt}] max jobs [${JOB_MAX_CNT}]" if [ "${running_jobs_cnt}" -ge "${JOB_MAX_CNT}" ]; then log_jc "Waiting for job to finish" wait_minus_n result=$? if [ "${result}" -ne 0 ]; then EXIT_CODE=1 # If a job fails, do not start any new ones. break fi fi log_jc "Starting new job" # from above; each array value is element/test_element. split it # back up element=${test%/*} test_element=${test#*/} element_dir=$DIB_ELEMENTS/${element}/test-elements/${test_element}/ # tests default to disk-based, but "element-type" can optionally # override that element_type=disk element_type_override=${element_dir}/element-type if [ -f ${element_type_override} ]; then element_type=$(cat ${element_type_override}) fi # override the output format if specified element_output=${DEFAULT_OUTPUT_FORMATS} element_output_override=${element_dir}/test-output-formats if [ -f $element_output_override ]; then element_output=$(cat ${element_output_override}) fi echo "Running $test ($element_type)" run_${element_type}_element_test $test_element $element ${DONT_USE_TMP} "${element_output}" & done # Wait for the rest of the jobs while true; do running_jobs_cnt=$(job_cnt) log_jc "Number of running jobs left [${running_jobs_cnt}]" if [ "${running_jobs_cnt}" -eq 0 ]; then break; fi wait_minus_n result=$? if [ "${result}" -ne 0 ]; then EXIT_CODE=1 fi done if [ "${EXIT_CODE}" -eq 0 ]; then echo "Tests passed!" exit 0 else echo "At least one test failed" exit 1 fi diskimage-builder-2.11.0/tests/run_output_format_test.sh000077500000000000000000000040301324273057700235170ustar00rootroot00000000000000#!/bin/bash set -eu set -o pipefail # # run_output_format_test.sh # # Use docker to test generation of various output formats. # BASE_DIR=$(cd $(dirname "$0")/.. && pwd) export DIB_ELEMENTS=$BASE_DIR/elements export TEST_ELEMENTS=$BASE_DIR/tests/elements export DIB_CMD=$BASE_DIR/bin/disk-image-create function build_test_image() { format=${1:-} if [ -n "$format" ]; then type_arg="-t $format" else type_arg= format="qcow2" fi dest_dir=$(mktemp -d) base_dest=$(basename $dest_dir) trap "rm -rf $dest_dir; sudo docker rmi $base_dest/image" EXIT ELEMENTS_PATH=$DIB_ELEMENTS:$TEST_ELEMENTS \ $DIB_CMD -x $type_arg --docker-target=$base_dest/image \ -o $dest_dir/image -n fake-os format=$(echo $format | tr ',' ' ') for format in $format; do if [ $format != 'docker' ]; then img_path="$dest_dir/image.$format" if ! [ -f "$img_path" ]; then echo "Error: No image with name $img_path found!" exit 1 else echo "Found image $img_path." fi else if ! sudo docker images | grep $base_dest/image ; then echo "Error: No docker image with name $base_dest/image found!" exit 1 else echo "Found docker image $base_dest/image" fi fi done trap EXIT rm -rf $dest_dir if sudo docker images | grep $base_dest/image ; then sudo docker rmi $base_dest/image fi } test_formats="tar tgz squashfs raw qcow2 docker aci" for binary in qemu-img docker mksquashfs; do if [ -z "$(type $binary)" ]; then echo "Warning: No $binary binary found, cowardly refusing to run tests." exit 1 fi done for format in '' $test_formats; do build_test_image $format echo "Test passed for output formats '$format'." done combined_format=$(echo $test_formats | tr ' ' ',') build_test_image $combined_format echo "Test passed for output format '$combined_format'." diskimage-builder-2.11.0/tox.ini000066400000000000000000000024761324273057700165120ustar00rootroot00000000000000[tox] envlist = py35,py27,pep8,pylint minversion = 2.0 skipsdist = True [testenv] usedevelop = True install_command = pip install {opts} {packages} deps= -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands= python setup.py test --slowest --testr-args='{posargs}' passenv= DUMP_CONFIG_GRAPH [testenv:pep8] commands = flake8 {posargs} dib-lint [testenv:pylint] commands = pylint --rcfile pylint.cfg diskimage_builder [testenv:venv] commands = {posargs} [testenv:func] envdir = {toxworkdir}/venv commands = {toxinidir}/tests/run_functests.sh {posargs} [testenv:cover] # NOTE: this is "setup.py test" (*not* testr) which is a pbr wrapper # around testr. This understands --coverage-package-name which we # need due to underscore issues. commands = python setup.py test --coverage --coverage-package-name diskimage_builder --testr-args='{posargs}' [testenv:docs] commands = python setup.py build_sphinx [testenv:releasenotes] commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html [flake8] ignore = E125,E126,E127,H202,H803 exclude = .venv,.git,.tox,dist,doc/build,doc/source,*lib/python*,*egg,build,conf.py