vmdebootstrap-1.4/0000755000175000017500000000000012650743704014157 5ustar neilneil00000000000000vmdebootstrap-1.4/setup.cfg0000644000175000017500000000007312650743704016000 0ustar neilneil00000000000000[egg_info] tag_build = tag_date = 0 tag_svn_revision = 0 vmdebootstrap-1.4/doc/0000755000175000017500000000000012650743704014724 5ustar neilneil00000000000000vmdebootstrap-1.4/doc/conf.py0000644000175000017500000001542512646142323016225 0ustar neilneil00000000000000# -*- coding: utf-8 -*- # # vmdebootstrap documentation build configuration file # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import subprocess import sys import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.append(os.path.abspath('..')) # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode'] # Configuration for sphinx.ext.todo todo_include_todos = True # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. # source_encoding = 'utf-8' # The master toctree document. master_doc = 'index' # General information about the project. project = 'vmdebootstrap' copyright = '2015 Neil Williams' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = subprocess.Popen(['python', 'setup.py', '-V'], cwd=r'..', stdout=subprocess.PIPE).stdout.read().rstrip() # The full version, including alpha/beta/rc tags. release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # today = '' # Else, today_fmt is used as the format for a strftime call. # today_fmt = '%B %d, %Y' # List of documents that shouldn't be included in the build. # unused_docs = [] # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = [] exclude_patterns = ['pages/reference-architecture', 'tables.rst'] # The reST default role (used for this markup: `text`) to use for all documents. # default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. # add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). # add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. # show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". # html_title = None # A shorter title for the navigation bar. Default is the same as html_title. # html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. # html_logo = "" # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. # html_favicon = "favicon.ico" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. # html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. # html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. # html_additional_pages = {} # If false, no module index is generated. # html_use_modindex = True # If false, no index is generated. # html_use_index = True # If true, the index is split into individual pages for each letter. # html_split_index = False # If true, links to the reST sources are added to the pages. html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. html_show_sphinx = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. # html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). # html_file_suffix = '' # Output file base name for HTML help builder. htmlhelp_basename = 'vmdebootstrapdocs' # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). # latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). # latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'vmdebootstrap.tex', 'vmdebootstrap', 'Neil Williams', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. # latex_use_parts = False # Additional stuff for the LaTeX preamble. # latex_preamble = '' # Documents to append as an appendix to all manuals. # latex_appendices = [] # If false, no module index is generated. # latex_use_modindex = True # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'http://docs.python.org/': None} vmdebootstrap-1.4/doc/Makefile0000644000175000017500000001273012617135212016357 0ustar neilneil00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: -rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/vmdebootstrap.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/vmdebootstrap.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/vmdebootstrap" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/vmdebootstrap" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." vmdebootstrap-1.4/doc/index.rst0000644000175000017500000000014412617135212016554 0ustar neilneil00000000000000VMDebootstrap ############# .. toctree:: :maxdepth: 2 overview.rst live.rst devel.rst vmdebootstrap-1.4/doc/live.rst0000644000175000017500000001476512635272114016425 0ustar neilneil00000000000000vmdebootstrap for creation of live images ######################################### Role of vmdebootstrap ********************* ``vmdebootstrap`` is limited to the role of generating the rootfs for the live image - the architecture-specific part. ``vmdebootstrap`` then copies the kernel files out of the rootfs and runs ``mksquashfs``. The files in the directory specified by the ``--squash`` option are not themselves sufficient to create a live image. Remaining steps include configuration of grub and EFI, addition of other components (like a menu or Debian Installer) and packaging up into a isohybrid image. vmdebootstrap features ********************** Architecture support ==================== vmdebootstrap has explicit support for foreign architecture bootstraps using qemu static binformat handling as well as support for Debian releases from wheezy onwards. * This is **not** intended to provide support for all packages in the Debian archive. Some packages do not install correctly with binfmt handling and ``vmdebootstrap`` should be run natively when the package list is to include these packages. Whether to use the binfmt_handler or build natively depends on: #. the availability of a working default kernel for the images built for that architecture and how to configure the bootloader(s) to provide the relevant dtb where needed. #. the complexity of the package set and compatibility with configuring those packages using qemu-user. Some packages fail if the emulator cannot provide threading support or other mechanisms - package sets with such requirements would need to be built natively. Test with a smaller package set where possible. live-support package ==================== vmdebootstrap can support adding specific packages but a simpler approach is to use the existing task-* packages and only add packages manually where explicitly needed for a live image, using the ``live-support`` package. Running vmdebootstrap for debian-cd =================================== debian-cd runs vmdebootstrap inside a VM in a similar manner to how debian-live currently operates, as both debian-live and vmdebootstrap need to call debootstrap which involves making device nodes and needs to run as root. This outer VM is specific for the release of Debian being built. vmdebootstrap can build older releases and it may be necessary to use a newer version of vmdebootstrap than is present in jessie to build jessie and to use that version to build wheezy. Remember to use ``http://cdbuilder.debian.org/debian/`` for the bootstrap operations (--mirror option) and ``http://httpredir.debian.org/debian`` for the mirror to be used after the image has booted (--apt-mirror option). Ensure that a user is created (``--user 'user/live'``) and that ``sudo`` is added to the set of packages to install and the --sudo option is passed to vmdebootstrap to ensure that the user is added to the sudo group. The root user password should also be locked (--lock-root-password). * Consider using a blank password and enforcing a password to be set upon login for those images which can support this. ``mksquashfs`` can fail without indication of why and when it does, the image file can be 4Kb or so of junk. ``vmdebootstrap`` will fail if the squashfs output is less than 1MB. This can occur if the drive runs out of space but squashfs does not report an error. Customisation hooks =================== vmdebootstrap uses a single config file per image type and each config file can have a single customisation script. The config file specifies the architecture of the image and the binformat handler for that architecture (if used), so the customisation hook script can be architecture-specific. Customisation hook scripts are shell scripts which will be passed a single parameter - the directory which represents the root directory of the final image. These scripts can use standard shell support to include other common functions or call out to utilities known to be installed in the outer VM running vmdebootstrap. Customisation hooks clearly need to live in a VCS - examples will be carried in the ``examples`` directory of ``vmdebootstrap`` and in the ``/usr/share/vmdebootstrap/examples`` directory. Working scripts based on these examples will likely be within the debian-cd git repo. Unlike standard vmdebootstrap example scripts, the scripts calling vmdebootstrap itself do not need to use sudo as the call is made inside the outer VM which already has root. Using sudo will work but will output a message: sudo: unable to resolve host JESSIE-debian-live-builder The building of live images doesn't appear to need changes in the vmdebootstrap package itself. The changes to isolinux to add the menu config, splash screen and to provide access to the install menus can all be done after the generation of the squashfs. Installing task packages using debootstrap **omits** ``Recommended`` packages, resulting in a much smaller image which is not expected for a live image. Task selection needs to be done in the customisation hook using the chroot command, at which point the default apt configuration will install the Recommends as well as the Depends packages. Ensure that the image size is big enough. Use the helpers --------------- :file:`vmdebootstrap` provides helpers for customisation hooks - typically you call a series at the start, do your customisations and call a parallel set before the customisation script finishes. See :ref:`customisation_hooks`. * :ref:`export_env` - When installing using apt in the customisation script, ensure that the debconf non-interactive settings are exported to prevent the install waiting for keyboard interaction:: ``DEBIAN_FRONTEND=noninteractive`` * :ref:`mount_proc` - The customisation script needs to mount proc (and possibly other locations like ``/sys/``, ``/dev/`` and ``/dev/pts/``) before starting the apt install. * :ref:`cleanup` - cleanup mountpoints at the end of the script. * Calls to apt should also not output the progress bar but the actual package installation steps should be logged. * :ref:`prepare_apt_source` - Move the image apt sources aside and set the cdimage apt source instead. Use ``http://cdbuilder.debian.org/debian/``. * :ref:`replace_apt_source` - At the end of the customisation hook, remove that source and replace the original. * :ref:`disable_daemons` - any daemons installed into the system need to know that the daemon should not be started until boot. * :ref:`remove_daemon_block` - allow installed daemons to start, once all package installations are complete. vmdebootstrap-1.4/doc/devel.rst0000644000175000017500000000770712617135440016563 0ustar neilneil00000000000000.. _customisation_hooks: Developing live scripts and customisation hooks =============================================== :file:`vmdebootstrap` is available in git and in Debian. The live image processing requires several options which are only available in versions of vmdebootstrap newer than version 0.5-2 available in Debian Jessie. vmdebootstrap is able to run on Stretch, Jessie or Wheezy and able to build any suite supported by debootstrap (and and architecture supported by QEMU) on any of those versions of Debian. This leads to a large matrix of build options and hooks. Calls to vmdebootstrap are best scripted. See the README for notes on which options and settings are required to make a live image using vmdebootstrap. The 'common' library contains functions and parameters which need to be used in *all* images, including:: export_env mount_proc disable_daemons prepare_apt_source replace_apt_source remove_daemon_block cleanup .. _cleanup: cleanup ------- Ensure that :file:`proc` is unmounted even if the customisation fails or else the image build itself will fail to unmount :file:`${rootdir}`. .. _export_env: export_env ---------- Debconf needs to be set in noninteractive mode to prevent the image build waiting for keyboard intervention. .. _mount_proc: mount_proc ---------- Many packages require ``/proc`` to be mounted inside the chroot during installation - cleanup must be specified as a trap if ``mount_proc`` is used:: trap cleanup 0 .. _disable_daemons: disable_daemons --------------- Packages which include a daemon **must not** start those daemons inside the chroot as this will make the ``${rootdir}`` appear busy and the unmount will fail. All scripts need to use :ref:`remove_daemon_block` after package installation is complete. .. _prepare_apt_source: prepare_apt_source ------------------ The final Debian mirror location is not useful during the build as there is a faster mirror available during the build. This function moves the specified mirror file aside and uses the nearby mirror. Always use with :ref:`replace_apt_source`. .. _remove_daemon_block: remove_daemon_block ------------------- After using :ref:`disable_daemons`, a policy script remains which needs to be removed to allow daemons to start normally when the image itself is booted. Use ``remove_daemon_block`` as the next step once package installation is complete. .. _replace_apt_source: replace_apt_source ------------------ Requires :ref:`prepare_apt_source` to have been run first, then undoes the change to the apt sources and cleans up. .. index: task .. _task_packages: TASK_PACKAGES ------------- Some task packages are useful to all images, these are specified here and should be included in the set of packages to be installed using all customisation scripts. .. index: extra .. _extra_packages: EXTRA_PACKAGES -------------- Packages which are not part of an existing task but which are useful for all images and should be included in the set of packages to be installed using all customisation scripts. .. _new_architectures: New architectures ----------------- The precursor to new architecture support is :file:`vmdebootstrap` support. A default :file:`vmdebootstrap` (with no customisation hook) will need to work and any changes to the settings (e.g. ``--no-kernel --package linux-myarch-flavour``) There is default support for some architectures in :file:`vmdebootstrap` (e.g. armhf architectures select the armmp kernel), such support depends on how many users would use the same kernel compared to the number of possible kernel flavours for that architecture. For a Debian LIVE image, **all** packages must exist in Debian. The package list also needs a review - some packages will simply not exist for the specified architecture. Some architecture-specific packages need to be added, so each architecture has a particular customisation hook script. Package names frequently change between releases, so the package selection needs to be suite specific as well. vmdebootstrap-1.4/doc/overview.rst0000644000175000017500000005520612650726053017332 0ustar neilneil00000000000000VMDebootstrap ############# .. index:: purpose .. _purpose: Purpose ******* vmdebootstrap is a helper to install basic Debian system into virtual disk image. It wraps **debootstrap**. You need to run :file:`vmdebootstrap` as root. If the ``--verbose`` option is not used, no output will be sent to the command line. If the ``--log`` option is not used, no output will be sent to any log files either. To use the image, you probably want to create a virtual machine using your preferred virtualization technology, such as :file:`kvm` or :file:`qemu`. Configure the virtual machine to use the image you've created. Then start the virtual machine and log into it via its console to configure it. The image has an empty root password and will not have networking configured by default. Set the root password before you configure networking. .. _synopsis: .. index:: synopsis Synopsis ******** :: $ sudo vmdebootstrap --image=FILE --size=SIZE [--mirror=URL] [--distribution=NAME] Options ======= --output=FILE write output to FILE, instead of standard output --verbose report what is going on --image=FILE put created disk image in FILE --size=SIZE create a disk image of size SIZE (1000000000) --tarball=FILE tar up the disk's contents in FILE --mirror=URL use MIRROR as package source (http://httpredir.debian.org/debian/) --arch=ARCH architecture to use (amd64) --- if using an architecture which the host system cannot execute, ensure the ``--foreign`` option is also used. --distribution=NAME release to use (defaults to stable). The release needs to be a valid Debian or Ubuntu release name or codename. --debootstrapopts=OPTS Supply options and arguments to ``debootstrap``, separated by spaces. e.g. ``--debootstrapopts="variant=buildd no-check-gpg components=main,contrib"``. See **debootstrap (1)** for more information. This option replaces the ``--variant`` support in previous versions. --package=PACKAGE install PACKAGE onto system --custom-package=DEB install package in DEB file onto system (not from mirror) --no-kernel do not install a linux package --kernel-package=PACKAGE If ``--no-kernel`` is not used and the auto-selection of the **linux-image-586** or **linux-image-armmp** or **linux-image-$ARCH** package is not suitable, the kernel PACKAGE name can be specified explicitly. --enable-dhcp enable DHCP on eth0 --root-password=PASSWORD set root password --customize=SCRIPT run SCRIPT after setting up system. If the script does not exist in the current working directory, :file:`/usr/share/vmdebootstrap/examples/` will be checked as a fallback. The script needs to be executable and is passed the root directory of the debootstrap as the only argument. Use chroot if you need to execute binaries within the debootstrap. --hostname=HOSTNAME set name to HOSTNAME (debian) --user=USERSTRING create USER with PASSWORD. The USERSTRING needs to be of the format: USER/PASSSWORD. --owner=OWNER change the owner of the final image from root to the specified user. --serial-console configure image to use a serial console (Wheezy only) --serial-console-command (Wheezy only.) Set the command to manage the serial console which will be appended to :file:`/etc/inittab`. Default is ``/sbin/getty \-L ttyS0 115200 vt100``, resulting in a line:: "S0:23:respawn:/sbin/getty \-L ttyS0 115200 vt100" --sudo install sudo, and if user is created, add them to sudo group --bootsize=BOOTSIZE If specified, create a /boot partition of the given size within the image. Debootstrapping will fail if this is too small for the selected kernel package and upgrading such a kernel package is likely to need two or three times the space of the installed kernel. --boottype=FSTYPE Filesystem to use for the /boot partition. (default ext2) --roottype=FSTYPE Filesystem to use for the / (root) partition. (default ext4) --swap=SWAPSIZE If specified, create a swap partition of the given size within the image. Debootstrapping will fail if this results in a root partition which is too small for the selected packages. The minimum swap space is 256MB as the default memory allocation of QEMU is 128MB. A default 1GB image is not likely to have enough space for a swap partition as well. --foreign=PATH Path to the binfmt_handler to enable foreign support in debootstrap. e.g. :file:`/usr/bin/qemu-arm-static` Note: foreign debootstraps may take a significant amount of time to complete and debootstrap will retry five times if packages fail to install by default. --no-extlinux Skip installation of extlinux. Needs a customize script or alternative bootloader to make the image bootable. Useful for architectures where extlinux is not supportable. Depending on how the image is to be booted, the ``--mbr`` option may also be necessary with extlinux. --squash=DIRECTORY Run mksquashfs against the rootfs using xz compression --- requires ``squashfs-tools`` to be installed. The squashfs and other files needed to use the squashfs to make a bootable system will be put into the specified directory. The directory will contain a ``filesystem.squashfs`` as well as the top level contents of the ``boot/`` directory. (If using UEFI, the ``boot/efi`` directory as well.) By default, ``mksquashfs`` is allowed to use all processors which may result in high load. squashfs can also have issues with large root filesystems. These errors can result in truncated files. This is a known bug in squashfs. ``vmdebootstrap`` will fail if the squashed filesystem is less than 1MB. --configure-apt Use the specified mirror and distribution to create a suitable apt source inside the VM. Can be useful if debootstrap fails to create it automatically. --apt-mirror Use the specified mirror inside the image instead of the mirror used to build the image. This is useful if you have a local mirror to make building the image quicker but the image needs to run even if that mirror is not available. Requires ``--configure-apt`` --grub Disable extlinux installation and configure grub2 instead. grub2 will be added to the list of packages to install. update-grub will be called once the debootstrap is complete and grub-install will be called in the image. --no-acpid Disable installation of acpid if not required, otherwise acpid will be installed if ``--foreign`` is not used. --sparse Skip optimizing image for compression and keep a sparse image. --pkglist Output a list of package names installed inside the image. Useful if you need to track the relevant source packages used inside the image for licence compliance. --dry-run Do not build, just test that the options are valid. --no-update-initramfs Skip the call to ``update-initramfs`` for reasons of speed or practicality. --convert-qcow2 Convert the final raw image to qcow2 format. --systemd-networkd Use Predictable Network Interface Names using systemd-networkd Configuration files and settings ******************************** --dump-config write out the entire current configuration --no-default-configs clear list of configuration files to read --config=FILE add FILE to config files Logging ******* --log=FILE write log entries to FILE (default is to not write log files at all); use "syslog" to log to system log, or "none" to disable logging. --log-level=LEVEL log at LEVEL, one of debug, info, warning, error, critical, fatal (default: debug). --log-max=SIZE rotate logs larger than SIZE, zero for never (default: 0) --log-keep=N keep last N logs (10) --log-mode=MODE set permissions of new log files to MODE (octal; default 0600) Performance *********** --dump-memory-profile=METHOD make memory profiling dumps using METHOD, which is one of: none, simple, meliae, or heapy (default: simple) --memory-dump-interval=SECONDS make memory profiling dumps at least SECONDS apart .. index:: networking .. _networking: Networking ********** Wheezy support ============== The ``--enable-networking`` option uses the :file:`/etc/network/interfaces.d/` source directory, with the default settings for ``lo`` and ``eth0`` being added to :file:`/etc/network/interfaces.d/setup`. Other networking configuration can be specified using a customisation script. Localhost settings would be:: auto lo iface lo inet loopback If ``--enable-dhcp`` is specified, these settings are also included into :file:`/etc/network/interfaces.d/setup`:: auto eth0 iface eth0 inet dhcp Jessie and later ================ In addition, ``systemd`` in jessie or later introduces PredictableNetworkInterfaceNames_ which are enabled using the ``systemd-networkd`` service. If this option is disabled, traditional interface names (like ``eth0``) will be used and the predictable names masked using ``udev``. Implementing the mask requires updating the initramfs, so the ``--update-initramfs`` option must not be disabled. If DHCP is also enabled, the following configuration is used:: /etc/systemd/network/99-dhcp.network ``systemd`` will use the first available match, so this can be overridden by putting another file into place using the customisation scripts, using a lower sorting filename. :: [Match] Name=en* [Network] DHCP=yes .. _PredictableNetworkInterfaceNames: http://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/ .. index:: bootloaders .. _bootloaders: Bootloaders *********** Unless the ``--no-extlinux`` or ``--grub`` options are specified, the image will use ``extlinux`` as a boot loader. ``bootsize`` is not recommended when using ``extlinux`` --- use ``grub`` instead. .. _wheezy_grub: Versions of grub2 in wheezy =========================== Grub2 in wheezy can fail to install in the VM, at which point :file:`vmdebootstrap` will fall back to ``extlinux``. It may still be possible to complete the installation of ``grub2`` after booting the VM as the problem may be related to the need to use loopback devices during the ``grub-install`` operation. Details of the error will appear in the vmdebootstrap log file, if enabled with the ``--log`` option. .. note:: **grub-legacy** is not supported. :file:`vmdebootstrap` also supports **EFI**. See :ref:`uefi`. Use ``--use-uefi`` to use ``grub-efi`` instead of ``grub-pc``. If the default 5MB is not enough space, use the ``--esp-size`` option to specify a different size for the EFI partition. Registered firmware is not supported as it would need to be done after boot. If the system you are creating is for more than just a VM or live image, you will likely need a larger ESP, up to 500MB. .. index: uefi .. _uefi: UEFI ==== UEFI support requires Grub and ``vmdebootstrap`` contains a configuration table of the UEFI components required for supported architectures. There are issues with running UEFI with QEMU on some architectures and a customisation script is available for amd64:: # vmdebootstrap --verbose --image jessie-uefi.img --grub --use-uefi \ --customize ./examples/qemu-efi-bochs-drm.sh ``vmdebootstrap`` supports UEFI for images and for squashfs but the necessary behaviour is different. With an image, an ESP vfat partition is created. With squashfs, the EFI files will be copied into an ``efi/`` directory in the squashfs output directory instead. There is EFI firmware available to use with QEMU when testing images built using the UEFI support, but this software is in Debian non-free due to patent concerns. If you choose to install ``ovmf`` to test UEFI builds, a secondary change is also needed to symlink the provided ``OVMF.fd`` to the file required by QEMU: ``bios-256k.bin`` and then tell QEMU about the location of this file with the -L option:: $ qemu-system-x86_64 -L /usr/share/ovmf/ -machine accel=kvm \ -m 4096 -smp 2 -drive format=raw,file=test.img To test the image, also consider using the ``qemu-wrapper.sh``:: $ /usr/share/vmdebootstrap/qemu-wrapper.sh jessie-uefi.img amd64 /usr/share/ovmf/ .. index: uboot .. _uboot: UBoot ===== UBoot needs manual configuration via the customisation hook scripts, typically support requires adding ``u-boot`` using ``--package`` and then copying or manipulating the relevant ``u-boot`` files in the customisation script. Examples are included for beaglebone-black. .. _installation_images: Installation images and virtual machines **************************************** :file:``vmdebootstrap`` is aimed principally at creating virtual machines, not installers or prebuilt installation images. It is possible to create prebuilt installation images for some devices but this depends on the specific device. (A 'prebuilt installation image' is a single image file which can be written to physical media in a single operation and which allows the device to boot directly into a fully installed system --- in a similar way to how a virtual machine would behave.) :file:`vmdebootstrap` assumes that all operations take place on a local image file or directory, not a physical block device / removable media. :file:`vmdebootstrap` is intended to be used with tools like ``qemu`` on the command line to launch a new virtual machine. Not all devices have virtualisation support in hardware. This has implications for :file:`u-boot` support in some cases. If the device can support reading the bootloader from a known partition, like the beaglebone-black, then :file:`vmdebootstrap` can provide space for the bootloader and the image will work as a prebuilt installation image. If the device expects that the bootloader exists at a specific offset and therefore requires that the bootloader is written as an image not as a binary which can be copied into an existing partition, :file:`vmdebootstrap` is unable to include that bootloader image into the virtual machine image. The beagleboneblack.sh script in the examples/ directory provides a worked example to create a prebuilt installation image. However, the beagleboneblack itself does not support virtualisation in hardware, so is unable to launch a virtual machine. Other devices, like the Cubietruck or Wandboard need :file:`u-boot` at a predefined offset but can launch a virtual machine using ``qemu``, so the cubietruck and wandboard6q scripts in the examples/ directory relate to building images for virtual machines once the device is already installed and booted into a suitable kernel. It is possible to wrap :file:`vmdebootstrap` in such a way as to prepare a physical block device with a bootloader image and then deploy the bootstrap on top. However, this does require physical media to be inserted and removed each time the wrapper is executed. To do this, use the ``--tarball`` option instead of the ``--image`` option. Then setup the physical media and bootloader image manually, as required for the device, redefine the partitions to make space for the rootfs, create a filesystem on the physical media and unpack the :file:`vmdebootstrap` tarball onto that filesystem. Once you have working media, an image can be created using dd to read back from the media to an image file, allowing other media to be written with a single image file. Example ******* To create an image for the stable release of Debian:: sudo vmdebootstrap --image test.img --size 1G \ --log test.log --log-level debug --verbose \ --mirror http://mirror.lan/debian/ To run the test image, make sure it is writeable. Use the ``--owner`` option to set mode 0644 for the specified user or use chmod manually:: sudo chmod a+w ./test.img Execute using qemu, e.g. on amd64 using qemu-system-x86_64:: qemu-system-x86_64 -drive format=raw,file=./test.img (This loads the image in a new window.) Note the use of ``-drive file=,format=raw`` which is needed for newer versions of QEMU. There is a ``bin/qemu-wrapper.sh `` script for simple calls where the ``--owner`` option is used, e.g.:: $ /usr/share/vmdebootstrap/qemu-wrapper.sh jessie.img amd64 There is EFI firmware available to use with QEMU when testing images built using the UEFI support, but this software is in Debian non-free due to patent concerns. If you choose to install ``ovmf`` to test UEFI builds, a secondary change is also needed to symlink the provided ``OVMF.fd`` to the file required by QEMU: ``bios-256k.bin`` and then tell QEMU about the location of this file with the -L option:: $ qemu-system-x86_64 -L /usr/share/ovmf/ -machine accel=kvm \ -m 4096 -smp 2 -drive format=raw,file=test.img For further examples, including u-boot support for beaglebone-black, see ``/usr/share/vmdebootstrap/examples`` Notes ***** If you get problems with the bootstrap process, run a similar bootstrap call directly and chroot into the directory to investigate the failure. The actual debootstrap call is part of the vmdebootstrap logfile. The debootstrap logfile, if any, will be copied into your current working directory on error. :file:`debootstrap` will download all the apt archive files into the apt cache and does not remove them before starting the configuration of the packages. This can mean that debootstrap can fail due to a lack of space on the device if the VM size is small. vmdebootstrap cleans up the apt cache once debootstrap has finished but this doesn't help if the package unpack or configuration steps use up all of the space in the meantime. Avoid this problem by specifying a larger size for the image. .. caution:: if you are also using a separate /boot partition in your options to :file:`vmdebootstrap` it may well be the boot partition which needs to be enlarged rather than the entire image. It is advisable to change the mirror in the example scripts to a mirror closer to your location, particularly if you need to do repeated builds. Use the --apt-mirror option to specify the apt mirror to be used inside the image, after boot. There are two types of examples for ARM devices available with :file:`vmdebootstrap`: prebuilt installation images (like the beaglebone-black) and virtual machine images (cubietruck and wandboard). ARM devices which do not support hypervisor mode and which also rely on the bootloader being at a specific offset instead of using a normal partition will **not** be supportable by vmdebootstrap. Similarly, devices which support hypervisor will only be supported using virtual machine images, unless the bootloader can be executed from a normal partition. .. index:: developing .. _developing: Developing ********** .. index:: pre-commit .. _pre_commit_hook: Testing vmdebootstrap from git ============================== ``vmdebootstrap`` uses ``yarn`` for the test suite, available in the `cmdtest `_ package. YARN is a scenario testing tool. Scenarios are written in mostly human readable language, however, they are not free form text. For more information on YARN see `the homepage `_:: $ sudo apt -y install cmdtest All commits must pass at least the fast tests. All merges into master need to pass a full test. All additions of new functionality must add fast and build tests --- fast tests for any new options and build tests which exercise the new functionality. Build tests can add checks for particular support on the machine running the test and skip if not found or add new environment settings to selectively run some build tests instead of all. If no arguments are given, the full test suite will be executed:: $ yarns/run-tests .. warning:: Do not run the full test suite if your connection to a Debian mirror is limited or metered. Each build requires a minimum of 2GB free space in tmpfs. A full test takes at least 10 minutes. When limiting the run to specific tests, each ``--env`` option needs to be specified separately:: $ sudo yarns/run-tests --env TESTS=build --env MIRROR=http://mirror/debian pre-commit ---------- All vmdebootstrap developers need to run the fast tests as a pre-commit hook --- any patches which fail this test will be rejected:: $ ln -s ../../pre-commit.sh .git/hooks/pre-commit The pre-commit hook just runs the fast tests which do not require ``sudo``. Fast tests ----------- The fast checks validate the handling of incompatible option arguments:: $ yarns/run-tests --env TESTS=fast Fast tests typically take a few seconds to run. Build tests ----------- The slow / build tests build multiple images and use ``sudo`` --- a local mirror is strongly recommended. :: $ sudo yarns/run-tests --env TESTS=build --env MIRROR=http://mirror/debian If ``MIRROR`` is not specified, a default mirror of ``http://httpredir.debian.org/debian/`` will be used. LAVA tests ---------- There is an example :file:`lava-submit.py` script which can be edited to automatically submit QEMU tests to a specified LAVA instance. The images themselves will use local ``file://`` URLs and therefore the ``lava-dispatcher`` needs to be installed locally. Configuring LAVA for these tests is a separate topic --- please ask on the `vmdebootstrap mailing list `_. vmdebootstrap-1.4/PKG-INFO0000644000175000017500000000124112650743704015252 0ustar neilneil00000000000000Metadata-Version: 1.1 Name: vmdebootstrap Version: 1.4 Summary: Bootstrap Debian into a (virtual machine) disk image Home-page: http://git.liw.fi/cgi-bin/cgit/cgit.cgi/vmdebootstrap/ Author: Neil Williams Author-email: codehelp@debian.org License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN Classifier: Development Status :: 5 - Production/Stable Classifier: Environment :: Console Classifier: Intended Audience :: Developers Classifier: Intended Audience :: System Administrators Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+) Classifier: Programming Language :: Python Classifier: Topic :: System :: Installation/Setup vmdebootstrap-1.4/setup.py0000644000175000017500000000325012645255026015670 0ustar neilneil00000000000000#!/usr/bin/env python # -*- coding: utf-8 -*- # # setup.py # # Copyright 2015 Neil Williams # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from setuptools import setup, find_packages setup( name='vmdebootstrap', version='1.4', description='Bootstrap Debian into a (virtual machine) disk image', author='Neil Williams', author_email='codehelp@debian.org', url='http://git.liw.fi/cgi-bin/cgit/cgit.cgi/vmdebootstrap/', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', 'Intended Audience :: Developers', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)', 'Programming Language :: Python', 'Topic :: System :: Installation/Setup', ], packages=[ 'vmdebootstrap', ], package_data={ 'vmdebootstrap': ['README', 'COPYING', 'NEWS'], }, install_requires=[ 'cliapp >= 1.20150829', 'distro-info', ], scripts=['bin/vmdebootstrap'] ) vmdebootstrap-1.4/MANIFEST.in0000644000175000017500000000041712645255026015716 0ustar neilneil00000000000000include NEWS include vmextract.py include vmsquash-tar.py include vmdebootstrap.8.in recursive-include examples *.sh *.txt *.py include common/* include doc/* include man/* recursive-include yarns *.yarn Makefile run-tests yarns.css shell.lib include bin/qemu-wrapper.sh vmdebootstrap-1.4/vmdebootstrap.egg-info/0000755000175000017500000000000012650743704020542 5ustar neilneil00000000000000vmdebootstrap-1.4/vmdebootstrap.egg-info/PKG-INFO0000644000175000017500000000124112650743703021634 0ustar neilneil00000000000000Metadata-Version: 1.1 Name: vmdebootstrap Version: 1.4 Summary: Bootstrap Debian into a (virtual machine) disk image Home-page: http://git.liw.fi/cgi-bin/cgit/cgit.cgi/vmdebootstrap/ Author: Neil Williams Author-email: codehelp@debian.org License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN Classifier: Development Status :: 5 - Production/Stable Classifier: Environment :: Console Classifier: Intended Audience :: Developers Classifier: Intended Audience :: System Administrators Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+) Classifier: Programming Language :: Python Classifier: Topic :: System :: Installation/Setup vmdebootstrap-1.4/vmdebootstrap.egg-info/SOURCES.txt0000644000175000017500000000242612650743704022432 0ustar neilneil00000000000000MANIFEST.in NEWS README setup.py vmextract.py bin/qemu-wrapper.sh bin/vmdebootstrap common/customise.lib common/jessie-amd64-hook.sh common/jessie-amd64.sh common/jessie-arm64-hook.sh common/jessie-arm64.sh doc/Makefile doc/conf.py doc/devel.rst doc/index.rst doc/live.rst doc/overview.rst examples/README.txt examples/beagleboneblack-customise.sh examples/beagleboneblack.sh examples/cubietruck-customise.sh examples/cubietruck.sh examples/jessie-uefi-amd64.sh examples/lava-submit.py examples/qemu-efi-bochs-drm.sh examples/vmdebootstrap.txt examples/wandboard-customise.sh examples/wandboard-uboot.sh examples/wandboard6q.sh man/Makefile man/conf.py man/index.rst man/overview.rst vmdebootstrap/__init__.py vmdebootstrap/base.py vmdebootstrap/codenames.py vmdebootstrap/constants.py vmdebootstrap/extlinux.py vmdebootstrap/filesystem.py vmdebootstrap/grub.py vmdebootstrap/network.py vmdebootstrap/uefi.py vmdebootstrap.egg-info/PKG-INFO vmdebootstrap.egg-info/SOURCES.txt vmdebootstrap.egg-info/dependency_links.txt vmdebootstrap.egg-info/requires.txt vmdebootstrap.egg-info/top_level.txt yarns/000-meta.yarn yarns/100-intro.yarn yarns/200-fast-tests.yarn yarns/300-slow-build-tests.yarn yarns/800-future.yarn yarns/900-implements.yarn yarns/Makefile yarns/run-tests yarns/shell.lib yarns/yarns.cssvmdebootstrap-1.4/vmdebootstrap.egg-info/top_level.txt0000644000175000017500000000001612650743703023270 0ustar neilneil00000000000000vmdebootstrap vmdebootstrap-1.4/vmdebootstrap.egg-info/dependency_links.txt0000644000175000017500000000000112650743703024607 0ustar neilneil00000000000000 vmdebootstrap-1.4/vmdebootstrap.egg-info/requires.txt0000644000175000017500000000004112650743703023134 0ustar neilneil00000000000000cliapp >= 1.20150829 distro-info vmdebootstrap-1.4/common/0000755000175000017500000000000012650743704015447 5ustar neilneil00000000000000vmdebootstrap-1.4/common/jessie-amd64.sh0000755000175000017500000000116112617135212020170 0ustar neilneil00000000000000#!/bin/sh set -e # define before sourcing common WHO=`whoami` USER='user/live' SUITE='jessie' SIZE='5G' ARCH='amd64' SHARE_PATH='/usr/share/vmdebootstrap/common' # needs a path for arch and task desktop IMAGE_PATH='.' . ${SHARE_PATH}/customise.lib sudo vmdebootstrap \ ${BASE_OPTS} --user ${USER} \ --size ${SIZE} \ --arch ${ARCH} \ --no-extlinux \ --grub --use-uefi \ --distribution ${SUITE} \ --customize "${SHARE_PATH}/${SUITE}-${ARCH}-hook.sh" \ --image ${IMAGE_PATH}/${SUITE}.img \ "$@" # report results and check we have something valid. ls -l ${IMAGE_PATH}/${SUITE}.img md5sum ${IMAGE_PATH}/${SUITE}.img vmdebootstrap-1.4/common/jessie-arm64-hook.sh0000755000175000017500000000123712617135212021150 0ustar neilneil00000000000000#!/bin/sh set -e rootdir=$1 # common needs rootdir to already be defined. . /usr/share/vmdebootstrap/common/customise.lib trap cleanup 0 mount_support disable_daemons prepare_apt_source chroot ${rootdir} apt-get -q -y install ${TASK_PACKAGES} ${EXTRA_PACKAGES} \ task-xfce-desktop exim4 mutt info rpcbind pciutils \ task-ssh-server task-print-server plymouth procmail \ m4 apt-listchanges at busybox nfs-common \ wamerican texinfo plymouth-themes plymouth-x11 uuid-runtime \ gettext-base mlocate irqbalance \ irqbalance user-setup zerofree remove_daemon_block replace_apt_source # particular to efi builds blacklist_qemu_bochs echo "Customisation complete" vmdebootstrap-1.4/common/customise.lib0000644000175000017500000000437412617135212020152 0ustar neilneil00000000000000# Copyright 2015 Neil Williams # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . BASE_OPTS=" --owner ${WHO} --verbose \ --sudo \ --lock-root-password \ --enable-dhcp \ --configure-apt \ --log debian-cd-${SUITE}-${ARCH}.log --log-level debug \ " TASK_PACKAGES="task-laptop task-english" # packages which are not (yet) part of a task EXTRA_PACKAGES="dkms locales whois telnet aptitude lsof host \ bash-completion firmware-linux-free dnsutils time rsync reportbug w3m \ ethtool ftp host lsof debian-faq debian-installer-launcher doc-debian \ " cleanup() { umount ${rootdir}/proc umount ${rootdir}/sys } export DEBIAN_FRONTEND=noninteractive export LC_ALL=C export LANG=C export LANGUAGE=C mount_support() { mount proc -t proc ${rootdir}/proc mount sys -t sysfs ${rootdir}/sys } disable_daemons() { # prevent packages starting daemons inside the chroot until after boot. # https://wiki.debian.org/chroot cat > ${rootdir}/usr/sbin/policy-rc.d < ${rootdir}/etc/apt/sources.list chroot ${rootdir} apt update } remove_daemon_block() { rm ${rootdir}/usr/sbin/policy-rc.d } replace_apt_source() { # Undo apt source change rm ${rootdir}/etc/apt/sources.list mv ${rootdir}/etc/apt/base.list ${rootdir}/etc/apt/sources.list.d/ } blacklist_qemu_bochs() { echo "blacklist bochs-drm" > ${rootdir}/etc/modprobe.d/qemu-blacklist.conf } vmdebootstrap-1.4/common/jessie-arm64.sh0000755000175000017500000000133612617135212020212 0ustar neilneil00000000000000#!/bin/sh set -e # define before sourcing common WHO=`whoami` USER='user/live' SUITE='jessie' SIZE='5G' ARCH='arm64' BINFMT='/usr/bin/qemu-aarch64-static' SHARE_PATH='/usr/share/vmdebootstrap/common' # needs a path for arch and task desktop IMAGE_PATH='.' . ${SHARE_PATH}/customise.lib sudo vmdebootstrap \ ${BASE_OPTS} --user ${USER} \ --size ${SIZE} \ --arch ${ARCH} \ --foreign ${BINFMT} \ --no-extlinux \ --grub --use-uefi \ --package dosfstools \ --distribution ${SUITE} \ --customize "${SHARE_PATH}/${SUITE}-${ARCH}-hook.sh" \ --image ${IMAGE_PATH}/${SUITE}-${ARCH}.img \ "$@" # report results and check we have something valid. ls -l ${IMAGE_PATH}/${SUITE}-${ARCH}.img md5sum ${IMAGE_PATH}/${SUITE}-${ARCH}.img vmdebootstrap-1.4/common/jessie-amd64-hook.sh0000755000175000017500000000133312617135212021127 0ustar neilneil00000000000000#!/bin/sh set -e rootdir=$1 # common needs rootdir to already be defined. . /usr/share/vmdebootstrap/common/customise.lib trap cleanup 0 mount_support disable_daemons # prepare_apt_source chroot ${rootdir} apt-get -q -y install ${TASK_PACKAGES} ${EXTRA_PACKAGES} \ task-xfce-desktop exim4 mutt info rpcbind pciutils \ task-ssh-server task-print-server plymouth procmail \ m4 open-vm-tools apt-listchanges at busybox nfs-common \ wamerican texinfo plymouth-themes plymouth-x11 uuid-runtime \ open-vm-tools-dkms open-vm-tools-desktop gettext-base mlocate \ irqbalance memtest86+ user-setup zerofree remove_daemon_block # replace_apt_source # particular to efi builds blacklist_qemu_bochs echo "Customisation complete" vmdebootstrap-1.4/bin/0000755000175000017500000000000012650743704014727 5ustar neilneil00000000000000vmdebootstrap-1.4/bin/qemu-wrapper.sh0000755000175000017500000000104112617135212017677 0ustar neilneil00000000000000#!/bin/sh set -e if [ -z "$1" ]; then echo "Usage: [uefi_directory]" echo "For x86_64, amd64 is also supported." exit 1 fi if [ -n "$2" ]; then if [ "$2" = 'amd64' ]; then ARCH='x86_64' else ARCH="$2" fi else echo "Specify the architecture of the image" echo "Usage: " echo "For x86_64, amd64 is also supported." exit 1 fi UEFI="" if [ -n "$3" ]; then UEFI="-L $3" fi qemu-system-${ARCH} -m 1024 ${UEFI} -enable-kvm -drive format=raw,file=./$1 vmdebootstrap-1.4/bin/vmdebootstrap0000755000175000017500000005643412650735552017564 0ustar neilneil00000000000000#! /usr/bin/python # Copyright 2011-2013 Lars Wirzenius # Copyright 2012 Codethink Limited # Copyright 2014-2015 Neil Williams # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import os import sys import time import shutil import cliapp import logging import datetime import tempfile import subprocess from vmdebootstrap.base import ( Base, runcmd, cleanup_apt_cache, ) from vmdebootstrap.grub import GrubHandler from vmdebootstrap.extlinux import ExtLinux from vmdebootstrap.codenames import Codenames from vmdebootstrap.filesystem import Filesystem from vmdebootstrap.uefi import Uefi from vmdebootstrap.network import Networking __version__ = '1.4' # pylint: disable=invalid-name,line-too-long # pylint: disable=missing-docstring,too-many-statements class VmDebootstrap(cliapp.Application): # pylint: disable=too-many-public-methods def __init__(self, progname=None, version=__version__, description=None, epilog=None): super(VmDebootstrap, self).__init__(progname, version, description, epilog) self.remove_dirs = [] self.mount_points = [] self.bootdir = None self.include = [] self.handlers = { Base.name: Base(), Uefi.name: Uefi(), Codenames.name: Codenames(), GrubHandler.name: GrubHandler(), ExtLinux.name: ExtLinux(), Filesystem.name: Filesystem(), Networking.name: Networking(), } def add_settings(self): # deliberately long lines for clarity. default_arch = subprocess.check_output(["dpkg", "--print-architecture"]).decode('utf-8').strip() self.settings.boolean(['verbose'], 'report what is going on') self.settings.string(['image'], 'put created disk image in FILE', metavar='FILE') self.settings.bytesize(['size'], 'create a disk image of size SIZE (%default)', metavar='SIZE', default='1G') self.settings.bytesize(['bootsize'], 'create boot partition of size ' 'SIZE (%default)', metavar='BOOTSIZE', default='0%') self.settings.string(['boottype'], 'specify file system type for /boot/', default='ext2') self.settings.bytesize(['bootoffset'], 'Space to leave at start of the ' 'image for bootloader', default='0') self.settings.boolean(['use-uefi'], 'Setup image for UEFI boot', default=False) self.settings.bytesize(['esp-size'], 'Size of EFI System Partition - ' 'requires use-uefi', default='5mib') self.settings.string(['part-type'], 'Partition type to use for this image', default='msdos') self.settings.string(['roottype'], 'specify file system type for /', default='ext4') self.settings.bytesize(['swap'], 'create swap space of size SIZE (min 256MB)') self.settings.string(['foreign'], 'set up foreign debootstrap environment ' 'using provided program (ie binfmt handler)') self.settings.string_list(['debootstrapopts'], 'pass additional options to debootstrap') self.settings.boolean(['extlinux'], 'install extlinux?', default=True) self.settings.string(['tarball'], "tar up the disk's contents in FILE", metavar='FILE') self.settings.string(['apt-mirror'], 'configure apt to use MIRROR', metavar='URL') self.settings.string(['mirror'], 'use MIRROR as package source (%default)', metavar='URL', default='http://httpredir.debian.org/debian/') self.settings.string(['arch'], 'architecture to use (%default)', metavar='ARCH', default=default_arch) self.settings.string(['distribution'], 'release to use (%default)', metavar='NAME', default='stable') self.settings.string_list(['package'], 'install PACKAGE onto system') self.settings.string_list(['custom-package'], 'install package in DEB file ' 'onto system (not from mirror)', metavar='DEB') self.settings.boolean(['no-kernel'], 'do not install a linux package') self.settings.string(['kernel-package'], 'install PACKAGE instead of ' 'the default kernel package', metavar='PACKAGE') self.settings.boolean(['enable-dhcp'], 'enable DHCP') self.settings.string(['root-password'], 'set root password', metavar='PASSWORD') self.settings.boolean(['lock-root-password'], 'lock root account so they ' 'cannot login?') self.settings.string(['customize'], 'run SCRIPT after setting up system', metavar='SCRIPT') self.settings.string(['hostname'], 'set name to HOSTNAME (%default)', metavar='HOSTNAME', default='debian') self.settings.string_list(['user'], 'create USER with PASSWORD', metavar='USER/PASSWORD') self.settings.boolean(['serial-console'], 'configure image to use a serial console') self.settings.string(['serial-console-command'], 'command to manage the ' 'serial console, appended to /etc/inittab (%default)', metavar='COMMAND', default='/sbin/getty -L ttyS0 115200 vt100') self.settings.boolean(['sudo'], 'install sudo, and if user is created, ' 'add them to sudo group') self.settings.string(['owner'], 'the user who will own the image when ' 'the build is complete.') self.settings.string(['squash'], 'use squashfs on the rootfs - ' 'cannot be used with --image', metavar='DIRECTORY') self.settings.boolean(['configure-apt'], 'Create an apt source based on ' 'the distribution and mirror selected.') self.settings.boolean(['mbr'], 'Run install-mbr (default if extlinux used)') self.settings.boolean(['grub'], 'Install and configure grub2 - ' 'disables extlinux.') self.settings.boolean(['sparse'], 'Do not fill the image with zeros to ' 'keep a sparse disk image', default=False) self.settings.boolean(['pkglist'], 'Create a list of package names ' 'included in the image.') self.settings.boolean(['no-acpid'], 'do not install the acpid package', default=False) self.settings.boolean(['update-initramfs'], 'Run update-initramfs after customisation', default=True) self.settings.boolean(['convert-qcow2'], 'Convert final image to qcow2', default=False) self.settings.boolean(['systemd-networkd'], 'Use Predictable Network ' 'Interface Names', default=True) self.settings.boolean(['dry-run'], 'do not build, just test the options', default=False) def process_args(self, args): # pylint: disable=too-many-branches,too-many-statements for _, handler in list(self.handlers.items()): handler.define_settings(self.settings) distro = self.handlers[Codenames.name] if self.settings['squash'] and self.settings['image']: raise cliapp.AppException( '--image can no longer be used with --squash') if not self.settings['image'] and not ( self.settings['tarball'] or self.settings['squash']): raise cliapp.AppException( 'You must give disk image filename or use either a ' 'tarball filename or use squash') if self.settings['image'] and not self.settings['size']: raise cliapp.AppException( 'If disk image is specified, you must give image size.') if not distro.debian_info.valid(self.settings['distribution']): if not distro.ubuntu_info.valid(self.settings['distribution']): raise cliapp.AppException( '%s is not a valid Debian or Ubuntu suite or codename.' % self.settings['distribution']) if self.settings['apt-mirror'] and not self.settings['configure-apt']: raise cliapp.AppException( '--apt-mirror requires --configure-apt as well.') if self.settings['convert-qcow2'] and not self.settings['image']: raise cliapp.AppException( '--convert-qcow2 can only be used with --image.') if self.settings['image'] and self.settings['tarball']: raise cliapp.AppException( 'Use --image or --tarball, not both.') if self.settings['squash'] and self.settings['tarball']: raise cliapp.AppException( 'Use --squash or --tarball, not both.') if not distro.was_oldstable(datetime.date(2015, 4, 26)): if not self.settings['systemd-networkd'] and\ not self.settings['update-initramfs']: raise cliapp.AppException( 'Disabling systemd-networkd for jessie and later ' 'requires updating the initramfs.') if self.settings['roottype'] == 'btrfs' and\ self.settings['extlinux'] and not self.settings['grub']: raise cliapp.AppException( 'extlinux is not yet supported on btrfs. Use grub.') uefi = self.handlers[Uefi.name] oldstable = distro.was_oldstable(datetime.date(2015, 4, 26)) uefi.check_settings(oldstable=oldstable) self.include = self._bootstrap_packages() if self.settings['dry-run']: print("Selected packages: ", ', '.join(self.include)) sys.exit(0) if os.geteuid() != 0: sys.exit("You need to have root privileges to run this script.") self.start_ops() def _image_preparations(self): uefi = self.handlers[Uefi.name] base = self.handlers[Base.name] filesystem = self.handlers[Filesystem.name] extlinux = self.handlers[ExtLinux.name] base.create_empty_image() self.partition_image() extlinux.install_mbr() filesystem.setup_kpartx() rootdev = filesystem.devices['rootdev'] roottype = filesystem.devices['roottype'] bootdev = filesystem.devices['bootdev'] if self.settings['swap'] > 0: base.message("Creating swap space") runcmd(['mkswap', filesystem.devices['swapdev']]) filesystem.mkfs(rootdev, fstype=roottype) rootdir = self.mount(rootdev) filesystem.devices['rootdir'] = rootdir if self.settings['use-uefi']: self.bootdir = uefi.prepare_esp(rootdir, bootdev) logging.debug("mounting %s", self.bootdir) self.mount(bootdev, self.bootdir) logging.debug(runcmd(['mount'])) elif bootdev: boottype = self.settings['boottype'] filesystem.mkfs(bootdev, fstype=boottype) self.bootdir = '%s/%s' % (rootdir, 'boot/') filesystem.devices['bootdir'] = self.bootdir os.mkdir(self.bootdir) self.mount(bootdev, self.bootdir) def _image_operations(self, rootdir, rootdev): if not self.settings['image']: return logging.debug("rootdir=%s rootdev=%s", rootdir, rootdev) grub = self.handlers[GrubHandler.name] extlinux = self.handlers[ExtLinux.name] base = self.handlers[Base.name] uefi = self.handlers[Uefi.name] distro = self.handlers[Codenames.name] if self.settings['use-uefi']: bootdir = self.bootdir logging.debug( "rootdir=%s rootdev=%s bootdir=%s", rootdir, rootdev, bootdir) logging.debug(runcmd(['mount'])) if not os.path.ismount(bootdir): logging.warning("%s had to be remounted", bootdir) self.mount(bootdir) grub.install_grub_uefi(rootdir) uefi.configure_efi(rootdir) grub.install_extra_grub_uefi(rootdir) uefi.configure_extra_efi(rootdir) elif self.settings['grub']: if not grub.install_grub2(rootdev, rootdir): extlinux.install_extlinux(rootdev, rootdir) elif self.settings['extlinux']: extlinux.install_extlinux(rootdev, rootdir) # only append for wheezy (which became oldstable on 2015.04.25) if distro.was_oldstable(datetime.date(2015, 4, 26)): base.append_serial_console(rootdir) elif self.settings['serial-console'] and not self.settings['grub']: base.message("Skipping setting serial console- wheezy only.") self.optimize_image(rootdir) def start_ops(self): base = self.handlers[Base.name] filesystem = self.handlers[Filesystem.name] network = self.handlers[Networking.name] distro = self.handlers[Codenames.name] try: if self.settings['image']: self._image_preparations() rootdir = filesystem.devices['rootdir'] rootdev = filesystem.devices['rootdev'] else: rootdir = self.mkdtemp() filesystem.devices['rootdir'] = rootdir rootdev = filesystem.devices['rootdev'] logging.debug("rootdir=%s rootdev=%s", rootdir, rootdev) self.debootstrap(rootdir) filesystem.set_hostname() filesystem.create_fstab() self.install_debs(rootdir) base.set_root_password(rootdir) base.create_users(rootdir) filesystem.remove_udev_persistent_rules() if distro.was_oldstable(datetime.date(2015, 4, 26)): network.setup_wheezy_networking(rootdir) else: network.setup_networking(rootdir) network.systemd_support(rootdir) filesystem.configure_apt() base.customize(rootdir) cleanup_apt_cache(rootdir) filesystem.update_initramfs() self._image_operations(rootdir, rootdev) filesystem.list_installed_pkgs() if self.settings['foreign']: os.unlink( '%s/usr/bin/%s' % (rootdir, os.path.basename(self.settings['foreign']))) if self.settings['tarball']: base.create_tarball(rootdir) elif self.settings['squash']: filesystem.squash_rootfs() filesystem.chown() filesystem.convert_image_to_qcow2() except BaseException as e: base.message('EEEK! Something bad happened...') rootdir = filesystem.devices['rootdir'] if rootdir: db_log = os.path.join(rootdir, 'debootstrap', 'debootstrap.log') if os.path.exists(db_log): shutil.copy(db_log, os.getcwd()) if self.settings['owner']: runcmd(["chown", self.settings["owner"], db_log]) base.message(e) self.cleanup_system() raise else: self.cleanup_system() def mkdtemp(self): dirname = tempfile.mkdtemp() self.remove_dirs.append(dirname) logging.debug('mkdir %s', dirname) return dirname def mount(self, device, path=None): base = self.handlers[Base.name] if not path: mount_point = self.mkdtemp() else: mount_point = path base.message('Mounting %s on %s' % (device, mount_point)) runcmd(['mount', device, mount_point]) self.mount_points.append(mount_point) logging.debug('mounted %s on %s', device, mount_point) return mount_point def partition_image(self): """ Uses fat16 (msdos) partitioning by default, use part-type to change. If bootoffset is specified, the first actual partition starts at that offset to allow customisation scripts to put bootloader images into the space, e.g. u-boot. """ base = self.handlers[Base.name] base.message('Creating partitions') uefi = self.handlers[Uefi.name] runcmd(['parted', '-s', self.settings['image'], 'mklabel', self.settings['part-type']]) partoffset = 0 extent = base.check_swap_size() # uefi uefi.partition_esp() # /boot partitioning offset calculation # returns partoffset if self.settings['bootoffset'] and self.settings['bootoffset'] is not '0': # turn v.small offsets into something at least possible to create. if self.settings['bootoffset'] < 1048576: partoffset = 1 logging.info( "Setting bootoffset %smib to allow for %s bytes", partoffset, self.settings['bootoffset']) else: partoffset = self.settings['bootoffset'] / (1024 * 1024) base.message( "Using bootoffset: %smib %s bytes" % (partoffset, self.settings['bootoffset'])) # /boot creation - move into base but keep the check # needs extent, partoffset, bootsize: no return if self.settings['bootsize'] and self.settings['bootsize'] is not '0%': if self.settings['grub'] and not partoffset: partoffset = 1 bootsize = self.settings['bootsize'] / (1024 * 1024) bootsize += partoffset base.message("Using bootsize %smib: %s bytes" % (bootsize, self.settings['bootsize'])) logging.debug("Starting boot partition at %sMB", bootsize) runcmd(['parted', '-s', self.settings['image'], 'mkpart', 'primary', 'fat16', str(partoffset), str(bootsize)]) logging.debug("Starting root partition at %sMB", partoffset) runcmd(['parted', '-s', self.settings['image'], 'mkpart', 'primary', str(bootsize), extent]) # uefi - make rootfs partition after end of ESP # needs extent elif self.settings['use-uefi']: uefi.make_root(extent) # no boot partition else: runcmd(['parted', '-s', self.settings['image'], 'mkpart', 'primary', '0%', extent]) # whatever we create, something needs the boot flag runcmd(['parted', '-s', self.settings['image'], 'set', '1', 'boot', 'on']) # return to doing swap setup base.make_swap(extent) def _bootstrap_packages(self): base = self.handlers[Base.name] uefi = self.handlers[Uefi.name] grub = self.handlers[GrubHandler.name] distro = self.handlers[Codenames.name] include = self.settings['package'] include.extend(base.base_packages()) include.extend(uefi.efi_packages()) include.extend(grub.grub_packages()) include.extend(distro.kernel_package()) return list(set(include)) def _debootstrap_second_stage(self, rootdir): base = self.handlers[Base.name] # set a noninteractive debconf environment for secondstage env = { "DEBIAN_FRONTEND": "noninteractive", "DEBCONF_NONINTERACTIVE_SEEN": "true", "LC_ALL": "C" } # add the mapping to the complete environment. env.update(os.environ) # First copy the binfmt handler over base.message('Setting up binfmt handler') shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir) # Next, run the package install scripts etc. base.message('Running debootstrap second stage') runcmd(['chroot', rootdir, '/debootstrap/debootstrap', '--second-stage'], env=env) def debootstrap(self, rootdir): base = self.handlers[Base.name] base.message( 'Debootstrapping %s [%s]' % ( self.settings['distribution'], self.settings['arch'])) args = ['debootstrap', '--arch=%s' % self.settings['arch']] if self.settings['package']: args.append( '--include=%s' % ','.join(self.include)) if self.settings['foreign']: args.append('--foreign') if self.settings['debootstrapopts']: for opt in self.settings['debootstrapopts']: for part in opt.split(' '): args.append('--%s' % part) logging.debug("debootstrap arguments: %s", args) args += [self.settings['distribution'], rootdir, self.settings['mirror']] logging.debug(" ".join(args)) runcmd(args) if self.settings['foreign']: self._debootstrap_second_stage(rootdir) def install_debs(self, rootdir): base = self.handlers[Base.name] if not self.settings['custom-package']: return base.message('Installing custom packages') tmp = os.path.join(rootdir, 'tmp', 'install_debs') os.mkdir(tmp) for deb in self.settings['custom-package']: shutil.copy(deb, tmp) filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb)) for deb in self.settings['custom-package']] out, err, _ = \ self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames) logging.debug('stdout:\n%s', out) logging.debug('stderr:\n%s', err) out = runcmd(['chroot', rootdir, 'apt-get', '-f', '--no-remove', 'install']) logging.debug('stdout:\n%s', out) shutil.rmtree(tmp) def optimize_image(self, rootdir): """ Filling up the image with zeros will increase its compression rate Can take an appreciable amount of time in very large images. """ if not self.settings['sparse']: base = self.handlers[Base.name] base.message("Optimizing image for compression") zeros = os.path.join(rootdir, 'ZEROS') self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M']) runcmd(['rm', '-f', zeros]) def cleanup_system(self): base = self.handlers[Base.name] # Clean up after any errors. base.message('Cleaning up') # Umount in the reverse mount order if self.settings['image']: for i in range(len(self.mount_points) - 1, -1, -1): mount_point = self.mount_points[i] try: runcmd(['umount', mount_point], ignore_fail=False) except cliapp.AppException: logging.debug("umount failed, sleeping and trying again") time.sleep(5) runcmd(['umount', mount_point], ignore_fail=False) runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True) for dirname in self.remove_dirs: shutil.rmtree(dirname) if __name__ == '__main__': VmDebootstrap(version=__version__).run() vmdebootstrap-1.4/vmdebootstrap/0000755000175000017500000000000012650743704017050 5ustar neilneil00000000000000vmdebootstrap-1.4/vmdebootstrap/codenames.py0000644000175000017500000000520712645255026021363 0ustar neilneil00000000000000""" Wrapper for distro information """ # -*- coding: utf-8 -*- # # codenames.py # # Copyright 2015 Neil Williams # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import datetime from vmdebootstrap.base import Base from distro_info import DebianDistroInfo, UbuntuDistroInfo # pylint: disable=missing-docstring class Codenames(Base): name = 'codenames' def __init__(self): super(Codenames, self).__init__() self.debian_info = DebianDistroInfo() self.ubuntu_info = UbuntuDistroInfo() self.settings = None def define_settings(self, settings): self.settings = settings def suite_to_codename(self, distro): suite = self.debian_info.codename(distro, datetime.date.today()) if not suite: return distro return suite def was_oldstable(self, limit): suite = self.suite_to_codename(self.settings['distribution']) # this check is only for debian if not self.debian_info.valid(suite): return False return suite == self.debian_info.old(limit) def was_stable(self, limit): suite = self.suite_to_codename(self.settings['distribution']) # this check is only for debian if not self.debian_info.valid(suite): return False return suite == self.debian_info.stable(limit) def kernel_package(self): packages = [] if self.settings['no-kernel'] or self.settings['kernel-package']: return packages if self.settings['arch'] == 'i386': # wheezy (which became oldstable on 04/25/2015) used '486' if self.was_oldstable(datetime.date(2015, 4, 26)): kernel_arch = '486' else: kernel_arch = '586' elif self.settings['arch'] == 'armhf': kernel_arch = 'armmp' elif self.settings['arch'] == 'ppc64el': kernel_arch = 'powerpc64le' else: kernel_arch = self.settings['arch'] packages.append('linux-image-%s' % kernel_arch) return packages vmdebootstrap-1.4/vmdebootstrap/extlinux.py0000644000175000017500000000735512617135212021304 0ustar neilneil00000000000000""" Wrapper for Extlinux support """ # -*- coding: utf-8 -*- # # extlinux.py # # Copyright 2015 Neil Williams # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import re import os import time import cliapp import logging from vmdebootstrap.base import Base, runcmd # pylint: disable=missing-docstring class ExtLinux(Base): name = 'extlinux' def __init__(self): super(ExtLinux, self).__init__() def install_extlinux(self, rootdev, rootdir): if not os.path.exists("/usr/bin/extlinux"): self.message("extlinux not installed, skipping.") return self.message('Installing extlinux') def find(pattern): dirname = os.path.join(rootdir, 'boot') basenames = os.listdir(dirname) logging.debug('find: %s', basenames) for basename in basenames: if re.search(pattern, basename): return os.path.join('boot', basename) raise cliapp.AppException('Cannot find match: %s' % pattern) try: kernel_image = find('vmlinuz-.*') initrd_image = find('initrd.img-.*') except cliapp.AppException as exc: self.message("Unable to find kernel. Not installing extlinux.") logging.debug("No kernel found. %s. Skipping install of extlinux.", exc) return out = runcmd(['blkid', '-c', '/dev/null', '-o', 'value', '-s', 'UUID', rootdev]) uuid = out.splitlines()[0].strip() conf = os.path.join(rootdir, 'extlinux.conf') logging.debug('configure extlinux %s', conf) kserial = 'console=ttyS0,115200' if self.settings['serial-console'] else '' extserial = 'serial 0 115200' if self.settings['serial-console'] else '' msg = ''' default linux timeout 1 label linux kernel %(kernel)s append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s %(extserial)s ''' % { 'kernel': kernel_image, # pylint: disable=bad-continuation 'initrd': initrd_image, # pylint: disable=bad-continuation 'uuid': uuid, # pylint: disable=bad-continuation 'kserial': kserial, # pylint: disable=bad-continuation 'extserial': extserial, # pylint: disable=bad-continuation } # pylint: disable=bad-continuation logging.debug("extlinux config:\n%s", msg) # python multiline string substitution is just ugly. # use an external file or live with the mangling, no point in # mangling the string to remove spaces just to keep it pretty in source. ext_f = open(conf, 'w') ext_f.write(msg) runcmd(['extlinux', '--install', rootdir]) runcmd(['sync']) time.sleep(2) def install_mbr(self): if not self.settings['mbr'] and not self.settings['extlinux']: return if os.path.exists("/sbin/install-mbr"): self.message('Installing MBR') runcmd(['install-mbr', self.settings['image']]) else: msg = "mbr enabled but /sbin/install-mbr not found" \ " - please install the mbr package." raise cliapp.AppException(msg) vmdebootstrap-1.4/vmdebootstrap/uefi.py0000644000175000017500000001472512635272114020356 0ustar neilneil00000000000000""" Wrapper for UEFI operations """ # -*- coding: utf-8 -*- # # uefi.py # # Copyright 2015 Neil Williams # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # pylint: disable=missing-docstring,duplicate-code import os import cliapp import logging from vmdebootstrap.base import ( Base, runcmd, mount_wrapper, umount_wrapper, ) from vmdebootstrap.constants import arch_table class Uefi(Base): name = 'uefi' def __init__(self): super(Uefi, self).__init__() self.bootdir = None def check_settings(self, oldstable=False): if not self.settings['use-uefi'] and self.settings['esp-size'] != 5242880: raise cliapp.AppException( 'You must specify use-uefi for esp-size to have effect') if self.settings['arch'] in arch_table and\ arch_table[self.settings['arch']]['exclusive'] and\ (not self.settings['use-uefi'] and not self.settings['squash']): raise cliapp.AppException( 'Only UEFI is supported on %s' % self.settings['arch']) elif self.settings['use-uefi'] and self.settings['arch'] not in arch_table: raise cliapp.AppException( '%s is not a supported UEFI architecture' % self.settings['arch']) if self.settings['use-uefi'] and ( self.settings['bootsize'] or self.settings['bootoffset']): raise cliapp.AppException( 'A separate boot partition is not supported with UEFI') if self.settings['use-uefi'] and not self.settings['grub']: raise cliapp.AppException( 'UEFI without Grub is not supported.') # wheezy (which became oldstable on 04/25/2015) only had amd64 uefi if oldstable: if self.settings['arch'] != 'amd64' and self.settings['use-uefi']: raise cliapp.AppException( 'Only amd64 supports UEFI in Wheezy') def efi_packages(self): packages = [] if not self.settings['use-uefi'] or\ self.settings['arch'] not in arch_table: return packages pkg = arch_table[self.settings['arch']]['package'] self.message("Adding %s to debootstrap" % pkg) packages.append(pkg) extra = arch_table[self.settings['arch']]['extra'] if extra and isinstance(extra, str): bin_pkg = arch_table[str(extra)]['bin_package'] self.message("Adding support for %s using %s" % (extra, bin_pkg)) packages.append(bin_pkg) return packages def copy_efi_binary(self, efi_removable, efi_install): if self.settings['arch'] not in arch_table: return logging.debug("using bootdir=%s", self.bootdir) if efi_removable.startswith('/'): efi_removable = efi_removable[1:] if efi_install.startswith('/'): efi_install = efi_install[1:] efi_output = os.path.join(self.bootdir, efi_removable) efi_input = os.path.join(self.bootdir, efi_install) logging.debug("moving %s to %s", efi_output, efi_input) if not os.path.exists(efi_input): logging.warning("%s does not exist (%s)", efi_input, efi_install) raise cliapp.AppException("Missing %s" % efi_input) if not os.path.exists(os.path.dirname(efi_output)): os.makedirs(os.path.dirname(efi_output)) logging.debug( 'Moving UEFI support: %s -> %s', efi_input, efi_output) if os.path.exists(efi_output): os.unlink(efi_output) os.rename(efi_input, efi_output) def configure_efi(self, rootdir): """ Copy the bootloader file from the package into the location so needs to be after grub and kernel already installed. """ if self.settings['arch'] not in arch_table: return self.message('Configuring EFI') mount_wrapper(rootdir) efi_removable = str(arch_table[self.settings['arch']]['removable']) efi_install = str(arch_table[self.settings['arch']]['install']) self.message('Installing UEFI support binary') logging.debug("moving %s to %s", efi_removable, efi_install) try: self.copy_efi_binary(efi_removable, efi_install) finally: umount_wrapper(rootdir) def configure_extra_efi(self, rootdir): if self.settings['arch'] not in arch_table: return extra = arch_table[self.settings['arch']]['extra'] if extra: mount_wrapper(rootdir) efi_removable = str(arch_table[extra]['removable']) efi_install = str(arch_table[extra]['install']) self.message('Copying UEFI support binary for %s' % extra) try: self.copy_efi_binary(efi_removable, efi_install) finally: umount_wrapper(rootdir) def partition_esp(self): if not self.settings['use-uefi']: return espsize = self.settings['esp-size'] / (1024 * 1024) self.message("Using ESP size: %smib %s bytes" % (espsize, self.settings['esp-size'])) runcmd(['parted', '-s', self.settings['image'], 'mkpart', 'primary', 'fat32', '1', str(espsize)]) runcmd(['parted', '-s', self.settings['image'], 'set', '1', 'boot', 'on']) runcmd(['parted', '-s', self.settings['image'], 'set', '1', 'esp', 'on']) def prepare_esp(self, rootdir, bootdev): self.bootdir = '%s/%s/%s' % (rootdir, 'boot', 'efi') logging.debug("bootdir:%s", self.bootdir) self.mkfs(bootdev, fstype='vfat') os.makedirs(self.bootdir) return self.bootdir def make_root(self, extent): bootsize = self.settings['esp-size'] / (1024 * 1024) + 1 runcmd(['parted', '-s', self.settings['image'], 'mkpart', 'primary', str(bootsize), extent]) vmdebootstrap-1.4/vmdebootstrap/filesystem.py0000644000175000017500000002720112650732660021607 0ustar neilneil00000000000000""" Wrapper for filesystem utilities """ # -*- coding: utf-8 -*- # # filesystem.py # # Copyright 2015 Neil Williams # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import os import cliapp import logging import tempfile from vmdebootstrap.base import ( Base, runcmd, copy_files ) # pylint: disable=missing-docstring class Filesystem(Base): name = 'filesystem' def __init__(self): super(Filesystem, self).__init__() self.settings = None self.devices = { 'rootdir': None, 'rootdev': None, 'bootdev': None, 'boottype': None, 'roottype': None, 'swapdev': None, } def define_settings(self, settings): self.settings = settings self.devices['roottype'] = self.settings['roottype'] def chown(self): if not self.settings['owner']: return # Change image owner after completed build if self.settings['image']: filename = self.settings['image'] elif self.settings['tarball']: filename = self.settings['tarball'] elif self.settings['squash']: filename = self.settings['squash'] else: return self.message("Changing owner to %s" % self.settings["owner"]) runcmd(["chown", "-R", self.settings["owner"], filename]) def update_initramfs(self): rootdir = self.devices['rootdir'] if not rootdir: raise cliapp.AppException("rootdir not set") if not os.path.exists( os.path.join(rootdir, 'usr', 'sbin', 'update-initramfs')): self.message("Error: Unable to run update-initramfs.") return if 'no-update-initramfs' in self.settings or not self.settings['update-initramfs']: return cmd = os.path.join('usr', 'sbin', 'update-initramfs') if os.path.exists(os.path.join(str(rootdir), cmd)): self.message("Updating the initramfs") runcmd(['chroot', rootdir, cmd, '-u']) def setup_kpartx(self): bootindex = None swapindex = None out = runcmd(['kpartx', '-avs', self.settings['image']]) if self.settings['bootsize'] and self.settings['swap'] > 0: bootindex = 0 rootindex = 1 swapindex = 2 parts = 3 elif self.settings['use-uefi']: bootindex = 0 rootindex = 1 parts = 2 elif self.settings['use-uefi'] and self.settings['swap'] > 0: bootindex = 0 rootindex = 1 swapindex = 2 parts = 3 elif self.settings['bootsize']: bootindex = 0 rootindex = 1 parts = 2 elif self.settings['swap'] > 0: rootindex = 0 swapindex = 1 parts = 2 else: rootindex = 0 parts = 1 boot = None swap = None devices = [line.decode('utf-8').split()[2] for line in out.splitlines() if line.decode('utf-8').startswith('add map ')] if len(devices) != parts: msg = 'Surprising number of partitions - check output of losetup -a' logging.debug("%s", runcmd(['losetup', '-a'])) logging.debug("%s: devices=%s parts=%s", msg, devices, parts) raise cliapp.AppException(msg) root = '/dev/mapper/%s' % devices[rootindex] if self.settings['bootsize'] or self.settings['use-uefi']: boot = '/dev/mapper/%s' % devices[bootindex].decode('utf-8') if self.settings['swap'] > 0: swap = '/dev/mapper/%s' % devices[swapindex] self.devices['rootdev'] = root self.devices['bootdev'] = boot self.devices['swap'] = swap def mkfs(self, device, fstype): self.message('Creating filesystem %s' % fstype) runcmd(['mkfs', '-t', fstype, device]) def create_fstab(self): rootdir = self.devices['rootdir'] rootdev = self.devices['rootdev'] bootdev = self.devices['bootdev'] boottype = self.devices['boottype'] roottype = self.devices['roottype'] def fsuuid(device): out = runcmd(['blkid', '-c', '/dev/null', '-o', 'value', '-s', 'UUID', device]) return out.splitlines()[0].strip() if rootdev: rootdevstr = 'UUID=%s' % fsuuid(rootdev) else: rootdevstr = '/dev/sda1' if bootdev and not self.settings['use-uefi']: bootdevstr = 'UUID=%s' % fsuuid(bootdev) else: bootdevstr = None if not rootdir: raise cliapp.AppException("rootdir not set") fstab = os.path.join(str(rootdir), 'etc', 'fstab') with open(fstab, 'w') as fstab: fstab.write('proc /proc proc defaults 0 0\n') fstab.write('%s / %s %s 0 1\n' % (rootdevstr, roottype, self.get_mount_flags(roottype))) if bootdevstr: fstab.write('%s /boot %s %s 0 2\n' % (bootdevstr, boottype, self.get_mount_flags(boottype))) if self.settings['swap'] > 0: fstab.write("/dev/sda3 swap swap defaults 0 0\n") elif self.settings['swap'] > 0: fstab.write("/dev/sda2 swap swap defaults 0 0\n") @staticmethod def get_mount_flags(fstype): """Return the fstab mount flags for a given file system type.""" flags = ['errors=remount-ro'] if fstype == 'btrfs': flags = [] return ','.join(flags) or 'defaults' def squash_rootfs(self): """ Run squashfs on the rootfs within the image. Copy the initrd and the kernel out, squashfs the rest. Also UEFI files, if enabled, ESP partition as a vfat image. TBD. """ if not self.settings['squash']: return if not os.path.exists('/usr/bin/mksquashfs'): logging.warning("Squash selected but mksquashfs not found!") return if not os.path.exists(self.settings['squash']): os.makedirs(self.settings['squash']) suffixed = os.path.join(self.settings['squash'], "filesystem.squashfs") if os.path.exists(suffixed): os.unlink(suffixed) _, exclusions = tempfile.mkstemp() with open(exclusions, 'w') as exclude: exclude.write("/proc\n") exclude.write("/dev\n") exclude.write("/sys\n") exclude.write("/run\n") self.message("Running mksquashfs on rootfs.") msg = runcmd( ['nice', 'mksquashfs', self.devices['rootdir'], suffixed, '-no-progress', '-comp', 'xz', '-e', exclusions], ignore_fail=False) os.unlink(exclusions) logging.debug(msg) check_size = os.path.getsize(suffixed) logging.debug("Created squashfs: %s", suffixed) if check_size < (1024 * 1024): logging.warning( "%s appears to be too small! %s bytes", suffixed, check_size) else: logging.debug("squashed size: %s", check_size) bootdir = os.path.join(self.devices['rootdir'], 'boot') # copying the boot/* files self.message("Copying boot files out of squashfs") copy_files(bootdir, self.settings['squash']) def configure_apt(self): rootdir = self.devices['rootdir'] if not self.settings['configure-apt'] or not self.settings['apt-mirror']: return if not rootdir: raise cliapp.AppException("rootdir not set") # use the distribution and mirror to create an apt source self.message("Configuring apt to use distribution and mirror") conf = os.path.join(str(rootdir), 'etc', 'apt', 'sources.list.d', 'base.list') logging.debug('configure apt %s', conf) mirror = self.settings['mirror'] if self.settings['apt-mirror']: mirror = self.settings['apt-mirror'] self.message("Setting apt mirror to %s" % mirror) os.unlink(os.path.join(str(rootdir), 'etc', 'apt', 'sources.list')) source = open(conf, 'w') line = 'deb %s %s main\n' % (mirror, self.settings['distribution']) source.write(line) line = '#deb-src %s %s main\n' % (mirror, self.settings['distribution']) source.write(line) source.close() # ensure the apt sources have valid lists runcmd(['chroot', rootdir, 'apt-get', '-qq', 'update']) def list_installed_pkgs(self): if not self.settings['pkglist']: return rootdir = self.devices['rootdir'] # output the list of installed packages for sources identification self.message("Creating a list of installed binary package names") out = runcmd(['chroot', rootdir, 'dpkg-query', '-W', "-f='${Package}.deb\n'"]) with open('dpkg.list', 'w') as dpkg: dpkg.write(out) def remove_udev_persistent_rules(self): rootdir = self.devices['rootdir'] if not rootdir: raise cliapp.AppException("rootdir not set") self.message('Removing udev persistent cd and net rules') for xrule in ['70-persistent-cd.rules', '70-persistent-net.rules']: pathname = os.path.join(str(rootdir), 'etc', 'udev', 'rules.d', xrule) if os.path.exists(pathname): logging.debug('rm %s', pathname) os.remove(pathname) else: logging.debug('not removing non-existent %s', pathname) def set_hostname(self): rootdir = self.devices['rootdir'] hostname = self.settings['hostname'] if not rootdir: raise cliapp.AppException("rootdir not set") with open(os.path.join(str(rootdir), 'etc', 'hostname'), 'w') as fhost: fhost.write('%s\n' % hostname) etc_hosts = os.path.join(str(rootdir), 'etc', 'hosts') try: with open(etc_hosts, 'r') as fhost: data = fhost.read() with open(etc_hosts, 'w') as fhosts: for line in data.splitlines(): if line.startswith('127.0.0.1'): line += ' %s' % hostname fhosts.write('%s\n' % line) except IOError: pass def make_rootfs_part(self, extent): bootsize = self.settings['esp-size'] / (1024 * 1024) + 1 runcmd(['parted', '-s', self.settings['image'], 'mkpart', 'primary', str(bootsize), extent]) def convert_image_to_qcow2(self): """ Current images are all prepared as raw rename to .raw and let the conversion put the original name back """ if not self.settings['convert-qcow2'] or not self.settings['image']: return self.message('Converting raw image to qcow2') tmpname = self.settings['image'] + '.raw' os.rename(self.settings['image'], tmpname) runcmd(['qemu-img', 'convert', '-O', 'qcow2', tmpname, self.settings['image']]) vmdebootstrap-1.4/vmdebootstrap/network.py0000644000175000017500000000721512645255026021117 0ustar neilneil00000000000000""" Wrapper for network support """ # -*- coding: utf-8 -*- # # network.py # # Copyright 2015 Neil Williams # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import os from vmdebootstrap.base import ( Base, runcmd, ) # pylint: disable=missing-docstring class Networking(Base): name = 'networking' def _write_network_interfaces(self, rootdir, line): self.message('Setting up networking') ifc_d = os.path.join(rootdir, 'etc', 'network', 'interfaces.d') ifc_file = os.path.join(rootdir, 'etc', 'network', 'interfaces') ethpath = os.path.join(ifc_d, 'setup') with open(ifc_file, 'w') as netfile: netfile.write(line) if not os.path.exists(ifc_d): os.mkdir(ifc_d) with open(ethpath, 'w') as eth: eth.write('auto lo\n') eth.write('iface lo inet loopback\n') if self.settings['enable-dhcp']: eth.write('\n') eth.write('auto eth0\n') eth.write('iface eth0 inet dhcp\n') def setup_wheezy_networking(self, rootdir): """ unconditionally write for wheezy (which became oldstable on 2015.04.25) """ self._write_network_interfaces( rootdir, 'source /etc/network/interfaces.d/*\n') def setup_networking(self, rootdir): self._write_network_interfaces( rootdir, 'source-directory /etc/network/interfaces.d\n') def systemd_support(self, rootdir): """ Handle the systemd-networkd setting """ if self.settings['systemd-networkd']: self.enable_systemd_networkd(rootdir) else: self.mask_udev_predictable_rules(rootdir) def mask_udev_predictable_rules(self, rootdir): """ This can be reset later but to get networking using eth0 immediately upon boot, the interface we're going to use must be known and must update the initramfs after setting up the mask. """ self.message('Disabling systemd predictable interface names') udev_path = os.path.join( 'etc', 'udev', 'rules.d', '80-net-setup-link.rules') runcmd(['chroot', rootdir, 'ln', '-s', '/dev/null', udev_path]) def enable_systemd_networkd(self, rootdir): """ Get networking working immediately on boot, allow any en* interface to be enabled by systemd-networkd using DHCP https://coreos.com/os/docs/latest/network-config-with-networkd.html http://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/ """ if not self.settings['enable-dhcp']: return self.message('Enabling systemd-networkd for DHCP') ethpath = os.path.join(rootdir, 'etc', 'systemd', 'network', '99-dhcp.network') with open(ethpath, 'w') as eth: eth.write('[Match]\n') eth.write('Name=en*\n') eth.write('\n[Network]\n') eth.write('DHCP=yes\n') runcmd(['chroot', rootdir, 'systemctl', 'enable', 'systemd-networkd']) vmdebootstrap-1.4/vmdebootstrap/__init__.py0000644000175000017500000000000012617135212021137 0ustar neilneil00000000000000vmdebootstrap-1.4/vmdebootstrap/base.py0000644000175000017500000001627312646142147020344 0ustar neilneil00000000000000""" Base for common utility functions """ # -*- coding: utf-8 -*- # # base.py # # Copyright 2015 Neil Williams # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import os import crypt import shutil import cliapp import logging import subprocess # pylint: disable=missing-docstring def runcmd(argv, stdin='', ignore_fail=False, env=None, **kwargs): logging.debug('runcmd: %s %s %s', argv, env, kwargs) proc = subprocess.Popen( argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, **kwargs) out, err = proc.communicate(stdin) if proc.returncode != 0: msg = 'command failed: %s\n%s\n%s' % (argv, out, err) logging.error(msg) if not ignore_fail: raise cliapp.AppException(msg) return out # FIXME: use contextmanager def mount_wrapper(rootdir): runcmd(['mount', '/dev', '-t', 'devfs', '-obind', '%s' % os.path.join(rootdir, 'dev')]) runcmd(['mount', '/dev/pts', '-t', 'devpts', '-obind', '%s' % os.path.join(rootdir, 'dev', 'pts')]) runcmd(['mount', '/proc', '-t', 'proc', '-obind', '%s' % os.path.join(rootdir, 'proc')]) runcmd(['mount', '/sys', '-t', 'sysfs', '-obind', '%s' % os.path.join(rootdir, 'sys')]) def umount_wrapper(rootdir): runcmd(['umount', os.path.join(rootdir, 'sys')]) runcmd(['umount', os.path.join(rootdir, 'proc')]) runcmd(['umount', os.path.join(rootdir, 'dev', 'pts')]) runcmd(['umount', os.path.join(rootdir, 'dev')]) def cleanup_apt_cache(rootdir): out = runcmd(['chroot', rootdir, 'apt-get', 'clean']) logging.debug('stdout:\n%s', out) def set_password(rootdir, user, password): encrypted = crypt.crypt(password, '..') runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user]) def delete_password(rootdir, user): runcmd(['chroot', rootdir, 'passwd', '-d', user]) def copy_files(src, dest): for filename in os.listdir(src): if os.path.isdir(filename) or os.path.islink(filename): continue shutil.copyfile( os.path.join(src, filename), os.path.join(dest, filename)) class Base(object): name = 'base' def __init__(self): super(Base, self).__init__() self.settings = None def define_settings(self, settings): self.settings = settings def message(self, msg): logging.info(msg) if self.settings['verbose']: print(msg) def create_empty_image(self): self.message('Creating disk image') runcmd(['qemu-img', 'create', '-f', 'raw', self.settings['image'], str(self.settings['size'])]) def create_tarball(self, rootdir): # Create a tarball of the disk's contents # shell out to runcmd since it more easily handles rootdir self.message('Creating tarball of disk contents') runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.']) def mkfs(self, device, fstype): self.message('Creating filesystem %s' % fstype) runcmd(['mkfs', '-t', fstype, device]) def set_root_password(self, rootdir): if self.settings['root-password']: self.message('Setting root password') set_password(rootdir, 'root', self.settings['root-password']) elif self.settings['lock-root-password']: self.message('Locking root password') runcmd(['chroot', rootdir, 'passwd', '-l', 'root']) else: self.message('Give root an empty password') delete_password(rootdir, 'root') def create_users(self, rootdir): def create_user(vmuser): runcmd(['chroot', rootdir, 'adduser', '--gecos', vmuser, '--disabled-password', vmuser]) if self.settings['sudo']: runcmd(['chroot', rootdir, 'adduser', vmuser, 'sudo']) for userpass in self.settings['user']: if '/' in userpass: user, password = userpass.split('/', 1) create_user(user) set_password(rootdir, user, password) else: create_user(userpass) delete_password(rootdir, userpass) def customize(self, rootdir): script = self.settings['customize'] if not script: return if not os.path.exists(script): example = os.path.join("/usr/share/vmdebootstrap/examples/", script) if not os.path.exists(example): self.message("Unable to find %s" % script) return script = example self.message('Running customize script %s' % script) logging.info("rootdir=%s", rootdir) try: with open('/dev/tty', 'w') as tty: cliapp.runcmd([script, rootdir, self.settings['image']], stdout=tty, stderr=tty) except IOError: logging.debug('tty unavailable, trying in headless mode.') subprocess.call([script, rootdir, self.settings['image']]) def append_serial_console(self, rootdir): if self.settings['serial-console']: serial_command = self.settings['serial-console-command'] logging.debug('adding getty to serial console') inittab = os.path.join(rootdir, 'etc/inittab') # to autologin, serial_command can contain '-a root' with open(inittab, 'a') as ftab: ftab.write('\nS0:23:respawn:%s\n' % serial_command) def check_swap_size(self): # swap - modifies extent extent = '100%' swap = 256 * 1024 * 1024 if self.settings['swap'] > 0: if self.settings['swap'] > swap: swap = self.settings['swap'] else: # minimum 256MB as default qemu ram is 128MB logging.debug("Setting minimum 256MB swap space") extent = "%s%%" % int(100 * (self.settings['size'] - swap) / self.settings['size']) return extent def make_swap(self, extent): if self.settings['swap'] > 0: logging.debug("Creating swap partition") runcmd([ 'parted', '-s', self.settings['image'], 'mkpart', 'primary', 'linux-swap', extent, '100%']) def base_packages(self): packages = [] if not self.settings['foreign'] and not self.settings['no-acpid']: packages.append('acpid') if self.settings['sudo']: packages.append('sudo') if not self.settings['no-kernel']: if self.settings['kernel-package']: packages.append(self.settings['kernel-package']) return packages vmdebootstrap-1.4/vmdebootstrap/grub.py0000644000175000017500000001271212635272114020357 0ustar neilneil00000000000000""" Wrapper for Grub operations """ # -*- coding: utf-8 -*- # # grub.py # # Copyright 2015 Neil Williams # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # pylint: disable=missing-docstring,duplicate-code import os import cliapp import logging from vmdebootstrap.base import ( Base, runcmd, mount_wrapper, umount_wrapper ) from vmdebootstrap.uefi import arch_table def grub_serial_console(rootdir): cmdline = 'GRUB_CMDLINE_LINUX_DEFAULT="console=tty0 console=tty1 console=ttyS0,38400n8"' terminal = 'GRUB_TERMINAL="serial gfxterm"' command = 'GRUB_SERIAL_COMMAND="serial --speed=38400 --unit=0 --parity=no --stop=1"' grub_cfg = os.path.join(rootdir, 'etc', 'default', 'grub') logging.debug("Allowing serial output in grub config %s", grub_cfg) with open(grub_cfg, 'a+') as cfg: cfg.write("# %s serial support\n" % os.path.basename(__file__)) cfg.write("%s\n" % cmdline) cfg.write("%s\n" % terminal) cfg.write("%s\n" % command) def link_uuid(rootdev): """ This is mainly to fix a problem in update-grub where /etc/grub.d/10_linux Checks if the $GRUB_DEVICE_UUID exists in /dev/disk/by-uuid and falls back to $GRUB_DEVICE if it doesn't. $GRUB_DEVICE is /dev/mapper/loopXpY (on docker) Creating the symlink ensures that grub consistently uses $GRUB_DEVICE_UUID when creating /boot/grub/grub.cfg """ if os.path.exists('/.dockerenv'): logging.info("Running in docker container") runcmd(['mkdir', '-p', '/dev/disk/by-uuid']) uuid = runcmd(['blkid', '-c', '/dev/null', '-o', 'value', '-s', 'UUID', rootdev]) uuid = uuid.splitlines()[0].strip() os.symlink(rootdev, os.path.join('/dev/disk/by-uuid', uuid)) def unlink_uuid(rootdev): """ Reset the link created with link_uuid. """ if os.path.exists('/.dockerenv'): uuid = runcmd(['blkid', '-c', '/dev/null', '-o', 'value', '-s', 'UUID', rootdev]) uuid = uuid.splitlines()[0].strip() os.remove(os.path.join('/dev/disk/by-uuid', uuid)) class GrubHandler(Base): name = 'grub' def __init__(self): super(GrubHandler, self).__init__() def install_grub2(self, rootdev, rootdir): self.message("Configuring grub2") # rely on kpartx using consistent naming to map loop0p1 to loop0 grub_opts = os.path.join('/dev', os.path.basename(rootdev)[:-2]) if self.settings['serial-console']: grub_serial_console(rootdir) logging.debug("Running grub-install with options: %s", grub_opts) mount_wrapper(rootdir) link_uuid(rootdev) try: runcmd(['chroot', rootdir, 'update-grub']) runcmd(['chroot', rootdir, 'grub-install', grub_opts]) except cliapp.AppException as exc: logging.warning(exc) self.message("Failed. Is grub2-common installed? Using extlinux.") umount_wrapper(rootdir) return False unlink_uuid(rootdev) umount_wrapper(rootdir) return True def install_grub_uefi(self, rootdir): ret = True self.message("Configuring grub-uefi") target = arch_table[self.settings['arch']]['target'] grub_opts = "--target=%s" % target logging.debug("Running grub-install with options: %s", grub_opts) mount_wrapper(rootdir) try: runcmd(['chroot', rootdir, 'update-grub']) runcmd(['chroot', rootdir, 'grub-install', grub_opts]) except cliapp.AppException as exc: logging.warning(exc) ret = False self.message( "Failed to configure grub-uefi for %s" % self.settings['arch']) finally: umount_wrapper(rootdir) if not ret: raise cliapp.AppException("Failed to install grub uefi") def install_extra_grub_uefi(self, rootdir): ret = True extra = arch_table[self.settings['arch']]['extra'] if extra: logging.debug("Installing extra grub support for %s", extra) mount_wrapper(rootdir) target = arch_table[extra]['target'] grub_opts = "--target=%s" % target self.message("Adding grub target %s" % grub_opts) try: runcmd(['chroot', rootdir, 'update-grub']) runcmd(['chroot', rootdir, 'grub-install', grub_opts]) except cliapp.AppException as exc: logging.warning(exc) ret = False self.message( "Failed to configure grub-uefi for %s" % extra) finally: umount_wrapper(rootdir) if not ret: raise cliapp.AppException("Failed to install extra grub uefi") def grub_packages(self): if self.settings['grub'] and not self.settings['use-uefi']: return ['grub-pc'] return [] vmdebootstrap-1.4/vmdebootstrap/constants.py0000644000175000017500000000347012617135212021432 0ustar neilneil00000000000000""" Constants which can be used by any handler """ # -*- coding: utf-8 -*- # # constants.py # # Copyright 2015 Neil Williams # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . arch_table = { # pylint: disable=invalid-name 'amd64': { 'removable': '/EFI/boot/bootx64.efi', # destination location 'install': '/EFI/debian/grubx64.efi', # package location 'package': 'grub-efi-amd64', # bootstrap package 'bin_package': 'grub-efi-amd64-bin', # binary only 'extra': 'i386', # architecture to add binary package 'exclusive': False, # only EFI supported for this arch. 'target': 'x86_64-efi', # grub target name }, 'i386': { 'removable': '/EFI/boot/bootia32.efi', 'install': '/EFI/debian/grubia32.efi', 'package': 'grub-efi-ia32', 'bin_package': 'grub-efi-ia32-bin', 'extra': None, 'exclusive': False, 'target': 'i386-efi', }, 'arm64': { 'removable': '/EFI/boot/bootaa64.efi', 'install': '/EFI/debian/grubaa64.efi', 'package': 'grub-efi-arm64', 'bin_package': 'grub-efi-arm64-bin', 'extra': None, 'exclusive': True, 'target': 'arm64-efi', } } vmdebootstrap-1.4/vmextract.py0000755000175000017500000001243512646142351016552 0ustar neilneil00000000000000#! /usr/bin/python # -*- coding: utf-8 -*- # # Copyright 2015 Neil Williams # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import os import sys import cliapp import guestfs import tarfile import logging __version__ = '0.1' __desc__ = "Helper to mount an image read-only and extract files" + \ " or directories." # pylint: disable=missing-docstring class VmExtract(cliapp.Application): # pylint: disable=too-many-public-methods """ Support for extracting useful content from VM images. For example, to assist in validation. """ def __init__( self, progname=None, version=__version__, description=__desc__, epilog=None): super(VmExtract, self).__init__( progname, version, description, epilog) self.guest_os = None self.mps = [] def add_settings(self): self.settings.boolean( ['verbose'], 'report what is going on') self.settings.string( ['image'], 'image to read', metavar='FILE') self.settings.string( ['directory'], 'directory to extract as a tarball.') self.settings.string_list( ['path'], 'path to the filename to extract - can repeat.') self.settings.boolean( ['boot'], 'mount the boot partition as well as root') self.settings.string( ['filename'], 'name of tarball containing the extracted directory', default='vmextract.tgz', metavar='FILE') # pylint: disable=too-many-branches,too-many-statements def process_args(self, args): if not self.settings['image']: raise cliapp.AppException( 'You must give an image to read') if not self.settings['directory'] and not self.settings['path']: raise cliapp.AppException( 'You must provide either a filename or directory ' 'to extract') try: self.prepare() self.mount_root() if self.settings['boot']: self.mount_boot() if self.settings['directory']: self.extract_directory() elif self.settings['path']: for path in self.settings['path']: self.download(path) except BaseException as exc: self.message('EEEK! Something bad happened... %s' % exc) sys.exit(1) def message(self, msg): logging.info(msg) if self.settings['verbose']: print(msg) def prepare(self): """ Initialise guestfs """ self.message("Preparing %s" % self.settings['image']) self.guest_os = guestfs.GuestFS(python_return_dict=True) self.guest_os.add_drive_opts( self.settings['image'], format="raw", readonly=1) # ensure launch is only called once per run self.guest_os.launch() drives = self.guest_os.inspect_os() self.mps = self.guest_os.inspect_get_mountpoints(drives[0]) def download(self, path): """ Copy a single file out of the image If a filename is not specified, use the basename of the original. """ filename = os.path.basename(path) self.message( "Extracting %s as %s" % (path, filename)) self.guest_os.download(path, filename) if not os.path.exists(filename): return RuntimeError("Download failed") def mount_root(self): """ Mounts the root partition to / """ root = [part for part in self.mps if part == '/'][0] if not root: raise RuntimeError("Unable to identify root partition") self.guest_os.mount_ro(self.mps[root], '/') def mount_boot(self): """ Mounts the /boot partition to a new /boot mountpoint """ boot = [part for part in self.mps if part == '/boot'][0] if not boot: raise RuntimeError("Unable to identify boot partition") if not self.guest_os.is_dir('/boot'): self.guest_os.mkmountpoint('/boot') self.guest_os.mount_ro(self.mps[boot], '/boot') def extract_directory(self): """ Create a tarball of a complete directory existing in the image. """ if not self.settings['filename']: self.settings['filename'] = 'vmextract.tgz' self.guest_os.tar_out( self.settings['directory'], self.settings['filename'], compress='gzip') if not tarfile.is_tarfile(self.settings['filename']): raise RuntimeError("Extraction failed") def main(): VmExtract(version=__version__).run() return 0 if __name__ == '__main__': main() vmdebootstrap-1.4/README0000644000175000017500000001657012635272114015043 0ustar neilneil00000000000000README for vmdebootstrap ======================== `debootstrap` installs a basic Debian system into a directory, for use with `chroot`(8). `vmdebootstrap` is a wrapper around it to install Debian into a disk image, which can be used with a virtual machine (such as KVM). See the manual page and `vmdebootstrap --help` for details on how to use the program. The manual page has an example. Limitations ----------- `vmdebootstrap` is aimed principally at creating virtual machines, not installers or prebuilt installation images. It is possible to create prebuilt installation images for some devices but this depends on the specific device. (A 'prebuilt installation image' is a single image file which can be written to physical media in a single operation and which allows the device to boot directly into a fully installed system - in a similar way to how a virtual machine would behave.) * `vmdebootstrap` assumes that all operations take place on a local image file, not a physical block device / removable media. * `vmdebootstrap` is intended to be used with tools like `qemu` on the command line to launch a new virtual machine. Not all devices have virtualisation support in hardware. This has implications for `u-boot` support in some cases. If the device can support reading the bootloader from a known partition, like the Beaglebone-black, then `vmdebootstrap` can provide space for the bootloader and the image will work as a prebuilt installation image. If the device expects that the bootloader exists at a specific offset and therefore requires that the bootloader is written as an image not as a binary which can be copied into an existing partition, `vmdebootstrap` is unable to include that bootloader image into the virtual machine image. It is possible to wrap `vmdebootstrap` in such a way as to prepare a *physical block device* with a bootloader image and then deploy the bootstrap on top. However, this does require physical media to be inserted and removed each time the wrapper is executed. Once you have working media, an image can be created using ``dd`` to read back from the media to an image file, allowing other media to be written with a single image file. To do this, use the `--tarball` option to `vmdebootstrap` instead of the `--image`` option. Then setup the physical media and bootloader image as required for the device, redefine the partitions to make space for the rootfs, create a filesystem on the physical media and unpack the `vmdebootstrap` tarball onto that filesystem. What you need ------------- In order to use vmdebootstrap, you'll need a few things: * debootstrap * extlinux * qemu-img (in the qemu-utils package in Debian) * parted * mbr * kpartx * python-cliapp (see http://liw.fi/cliapp/) * python-distro-info Testing vmdebootstrap from git ------------------------------ There is a strongly recommended git pre-commit hook available for vmdebootstrap development - it requires the ``cmdtest`` package:: ln -s ../../pre-commit.sh .git/hooks/pre-commit Running vmdebootstrap from git ------------------------------ $ sudo PYTHONPATH=. ./bin/vmdebootstrap This has changed slightly with version 1.0 with the need for PYTHONPATH to reference the module approach for support handlers. vmdebootstrap modules --------------------- The single vmdebootstrap script has been refactored to be the top level settings parser and validator and the point where the other modules (handlers) get to be called in a collaborative sequence. The new modules are an attempt to work with a DRY process as well as keeping the source code itself maintainable. Handler functions need to check settings at the start so that calls to the handlers can be retained in a simple flow. Where a function needs code from multiple handlers, that function needs to be in the vmdebootstrap script but these should, ideally, be single calls into dedicated calls from the relevant handlers which can return True|False or raise cliapp.AppException to affect subsequent flow. Handlers must NOT hook into other handlers, except Base or constants, only the vmdebootstrap script has the full set, so use function arguments to pass variables populated by different handlers. Wherever possible, large sections of new functionality need to be added as new handlers. pylint ------ When using pylint, the following option is advised: $ pylint --ignore-imports=y vmdebootstrap (Despite the name of the option, this only ignores imports when computing similarities and various handlers will end up needing similar imports, it makes no sense to complain about that.) Apart from that, vmdebootstrap uses pylint and contains comments to disable certain pylint checks in certain areas. pylint compatibility will make it easier to accept patches, just follow the existing pattern of pylint usage. pylint is far from perfect but can be helpful. Testing UEFI support -------------------- There is EFI firmware available to use with QEMU when testing images built using the UEFI support, but this software is in Debian non-free due to patent concerns. If you choose to use it to test UEFI builds, a secondary change is also needed to symlink the provided OVMF.fd to the file required by QEMU: bios-256k.bin and then tell QEMU about the location of this file with the -L option: $ qemu-system-x86_64 -L /usr/share/ovmf/ -machine accel=kvm \ -m 4096 -smp 2 -drive file=amd64.img,format=raw Note the use of -drive file=,format=raw which is needed for newer versions of QEMU. The vmextract helper -------------------- Once the image is built, various files can be generated or modified during the install operations and some of these files can be useful when testing the image. One example is the initrd built by the process of installing a Debian kernel. Rather than having to mount the image and copy the files manually, the vmextract helper can do it for you, without needing root privileges. $ /usr/share/vmdebootstrap/vmextract.py --verbose \ --image bbb/bbb-debian-armmp.img --boot \ --path /boot/initrd.img-3.14-2-armmp \ --path /lib/arm-linux-gnueabihf/libresolv.so.2 This uses python-guestfs (a Recommended package for vmdebootstrap) to prepare a read-only version of the image - in this case with the /boot partition also mounted - and copies files out into the current working directory. The integration test suite -------------------------- To run the vmdebootstrap integration test suite, you need the yarn tool, which is part of cmdtest. See and the cmdtest package in Debian. The command to run the tests is: sudo yarns/run-tests You can skip the slow tests that actually build images, by setting the `TESTS` variable: sudo yarns/run-tests --env TESTS=fast To format the test suite document: (needs pandoc installed) make -C yarns Legalese -------- Copyright 2011-2013 Lars Wirzenius Copyright 2012 Codethink Limited This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . vmdebootstrap-1.4/examples/0000755000175000017500000000000012650743704015775 5ustar neilneil00000000000000vmdebootstrap-1.4/examples/README.txt0000644000175000017500000000640612617135212017471 0ustar neilneil00000000000000Examples ======== The Freedombox project are using vmdebootstrap for ARM based images. http://anonscm.debian.org/cgit/freedombox/freedom-maker.git/ Those scripts have been adapted to work directly within vmdebootstrap as customise scripts in this examples directory. There are also examples which fold all of the options into a single script which just needs to be called with a size and an image name. The only required argument for each example is the image name. Beaglebone-black ---------------- /usr/share/vmdebootstrap/examples/beagleboneblack.sh --image bbb.img Examples will run vmdebootstrap with sudo, you may be asked for authentication. Arguments other than those already included in the shortcut can also be supplied, where required. e.g. --size, --variant, --package (can be specified multiple times), --hostname, --sudo, --root-password or --lock-root-password. CubieTruck ---------- Currently untested and lacking u-boot support. QEMU and EFI ------------ The bochs-drm kernel driver can be a problem when testing UEFI images, even headless ones, causing systemd to halt before a login prompt is offered. vmdebootstrap includes a simple customisation script which blacklists the bochs-drm module. Use, copy or extend this script for any image which uses UEFI and which should be testable using QEMU. To run UEFI with QEMU, the ovmf package needs to be installed from non-free (due to patent issues with VFAT) and the -L option used to QEMU to indicate the directory containing the EFI firmware to use. For amd64, the firmware installed by ovmf can need to be renamed (or symlinked) as /usr/share/ovmf/bios-256k.bin - then supply the -L option to QEMU: $ qemu-system-x86_64 -machine accel=kvm -m 4096 -smp 2 -drive format=raw,file=test.img -L /usr/share/ovmf/ debootstrap and task packages ----------------------------- debootstrap is designed to be a minimalist tool and vmdebootstrap wraps this support without substantial changes. Task packages are the simplest way to extend a minimal bootstrap to a more general purpose machine but there are limitations. debootstrap does not handle Recommended packages, so installing a task package using the --package support of vmdebootstrap (just as with the --include support of debootstrap itself) may result in a system with fewer packages installed than expected. Such systems can have the extra packages identified after boot using graphical tools like aptitude but to have all packages available during the creation of the image, a customisation hook is required. The hook simply needs to install the task package using apt instead of passing the task package to --package. This allows apt to do all the normal Recommends calculations and results in all of the extra packages being installed in one operation. However, the apt source used for this will be the apt source specified to vmdebootstrap for use after the system is booted, so you may also want to extend the hook to temporarily reinstate a local mirror (as used for the bootstrap phase) and put the other mirror back at the end of the hook. Examples of such hooks are available here: http://anonscm.debian.org/cgit/debian-cd/pettersson-live.git/tree/vmdebootstrap (These will need modification for other uses as the hooks expect a particular filesystem layout only useful for debian-cd.) vmdebootstrap-1.4/examples/wandboard6q.sh0000755000175000017500000000113712645255026020545 0ustar neilneil00000000000000#!/bin/sh set -e # Important: this example is to create a VM image for wandboard # To boot the device itself, u-boot support will need to be added # to a real block device. user=`whoami` sudo ./vmdebootstrap \ --owner ${user} --verbose \ --size 3G \ --mirror http://httpredir.debian.org/debian \ --log wandboard.log --log-level debug \ --arch armhf \ --foreign /usr/bin/qemu-arm-static \ --enable-dhcp \ --configure-apt \ --no-extlinux \ --grub \ --distribution sid \ --serial-console-command "/sbin/getty -L ttymxc0 115200 vt100" \ --customize "./examples/wandboard-customise.sh" \ "$@" vmdebootstrap-1.4/examples/beagleboneblack.sh0000755000175000017500000000077312645255026021422 0ustar neilneil00000000000000#!/bin/sh set -e user=`whoami` sudo vmdebootstrap \ --owner ${user} --verbose \ --mirror http://httpredir.debian.org/debian \ --log beaglebone-black.log --log-level debug \ --arch armhf \ --foreign /usr/bin/qemu-arm-static \ --enable-dhcp \ --configure-apt \ --no-extlinux \ --package u-boot \ --package dosfstools \ --distribution sid \ --serial-console-command "'/sbin/getty -L ttyO0 115200 vt100'" \ --customize "beagleboneblack-customise.sh" \ --bootsize 100mib --boottype vfat \ "$@" vmdebootstrap-1.4/examples/wandboard-customise.sh0000755000175000017500000000035112452247131022276 0ustar neilneil00000000000000#!/bin/sh set -e rootdir=$1 mkdir -p $rootdir/boot/dtbs cp $rootdir/usr/lib/linux-image-*-armmp/* $rootdir/boot/dtbs for module in ahci_platform ahci_imx sd-mod; do echo ${module} >> ${rootdir}/etc/initramfs-tools/modules done vmdebootstrap-1.4/examples/cubietruck.sh0000755000175000017500000000063112645255026020473 0ustar neilneil00000000000000#!/bin/sh set -e user=`whoami` sudo vmdebootstrap \ --owner ${user} --verbose \ --size 3G \ --mirror http://httpredir.debian.org/debian \ --log cubietruck.log --log-level debug \ --arch armhf \ --foreign /usr/bin/qemu-arm-static \ --enable-dhcp \ --configure-apt \ --distribution sid \ --serial-console-command "/sbin/getty -L ttyS0 115200 vt100" \ --customize "cubietruck-customise.sh" \ "$@" vmdebootstrap-1.4/examples/wandboard-uboot.sh0000755000175000017500000000435012617134742021424 0ustar neilneil00000000000000#!/bin/sh set -e # This script is experimental and incomplete. # Expects a tarball rootfs which includes a kernel, e.g. # wandboard6q.sh --tarball wandboard.tgz # wandboard-uboot.sh /dev/mmcblk0 wandboard.tgz device=$1 tarball=$2 if [ -z "${device}" ]; then echo "Block device not specified" exit fi if [ ! -f "${device}" ]; then echo "Specified device does not exist: ${device}" exit fi if [ -z "${tarball}" ]; then echo "Tarball not specified" exit fi if [ ! -f '/usr/lib/u-boot/wandboard_quad/u-boot.imx' ]; then echo "Unable to find wandboard u-boot file" exit fi # u-boot needs to be dd'd to the device, not a partition # but kpartx does not setup the device, just the partitions sudo dd if=/dev/zero of=${device} bs=1M count=10 sudo dd if=/usr/lib/u-boot/wandboard_quad/u-boot.imx of=${device} seek=1 conv=fsync bs=1k sudo sfdisk --in-order --Linux --unit M ${device} <<-__EOF__ 1,,0x83,- __EOF__ sudo mkfs.ext4 ${device}p1 -L rootfs dir=`mktemp -d` sudo mount ${Ddevice}p1 ${dir} sudo tar -xzf ${tarball} -C ${dir} # assumes a single partition deployment to SD card ver=$(basename `find $rootdir/lib/modules/ -maxdepth 1 -mindepth 1 -type d`) sudo touch ${dir}/uEnv.txt sudo chmod 666 ${dir}/uEnv.txt echo autoload=no > ${dir}/uEnv.txt echo initrd_high=0xffffffff >> ${dir}/uEnv.txt echo fdt_high=0xffffffff >> ${dir}/uEnv.txt echo kernel_addr_r=0x11000000 >> ${dir}/uEnv.txt echo initrd_addr_r=0x13000000 >> ${dir}/uEnv.txt echo fdt_addr_r=0x12000000 >> ${dir}/uEnv.txt echo mmcdev=0 >> ${dir}/uEnv.txt echo mmcpart=1 >> ${dir}/uEnv.txt echo ver=3.16.0-4-armmp >> ${dir}/uEnv.txt echo loadkernel=load mmc ${mmcdev}:${mmcpart} ${kernel_addr_r} boot/vmlinuz-${ver} >> ${dir}/uEnv.txt echo loadinitrd=load mmc ${mmcdev}:${mmcpart} ${initrd_addr_r} boot/initrd.img-${ver}.uboot; setenv initrd_size ${filesize} >> ${dir}/uEnv.txt echo loadfdt=load mmc ${mmcdev}:${mmcpart} ${fdt_addr_r} boot/dtbs/imx6q-wandboard.dtb >> ${dir}/uEnv.txt echo bootargs=console=ttymxc0,115200 root=/dev/mmcblk0p1 rootwait rw ip=dhcp >> ${dir}/uEnv.txt echo uenvcmd=run loadkernel; run loadinitrd; run loadfdt; bootz ${kernel_addr_r} ${initrd_addr_r} ${fdt_addr_r} >> ${dir}/uEnv.txt sudo chmod 644 ${dir}/uEnv.txt sudo umount ${dir} sudo rm -rf ${dir} vmdebootstrap-1.4/examples/cubietruck-customise.sh0000755000175000017500000000034712452247113022502 0ustar neilneil00000000000000#!/bin/sh set -e rootdir=$1 mkdir -p $rootdir/boot/dtbs cp $rootdir/usr/lib/linux-image-*-armmp/* $rootdir/boot/dtbs for module in phy-sun4i-usb ohci-platform; do echo ${module} >> ${rootdir}/etc/initramfs-tools/modules done vmdebootstrap-1.4/examples/lava-submit.py0000755000175000017500000000621412635272114020574 0ustar neilneil00000000000000#!/usr/bin/env python # -*- coding: utf-8 -*- # # lava-submit.py # # Copyright 2015 Neil Williams # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, # MA 02110-1301, USA. # # """ Example script - needs configuration (or use lava-tool). Expects to submit a pipeline job to a QEMU device, so ensure that the host supports these jobs. **This script is not to be expanded with argparse or CLI options for the username, token, hostname or image**. It is meant to be copied into something like jenkins to do the submission using hardcoded values, themselves hidden behind a UI. Other details like architecture, prompt and login information may need to come from a config file or command line. LAVA needs a serial console to tell whether the VM booted or not. Larger images will need the LAVA device to have more memory available. """ import os import yaml import xmlrpclib # Constants for each particular script configuration. USERNAME = "" TOKEN = "" HOSTNAME = "" IMAGE = "" ARCH = "" PROMPT = "" PASSWORD = "" # leave empty if no root password def job(image): """ Bare bones YAML job definition """ job_def = { 'actions': [{ 'deploy': {'images': {'rootfs': { 'image_arg': "-drive format=raw,file={rootfs}", "url": "file://%s" % image }}, 'os': 'debian', 'timeout': {'minutes': 5}, 'to': 'tmpfs'} }, { 'boot': { 'media': 'tmpfs', 'prompts': [PROMPT], 'auto_login': { "login_prompt": "login:", "username": "root" }, 'method': 'qemu'} }], 'device_type': 'qemu', 'job_name': 'vmdebootstrap-test', 'priority': 'medium', "context": {"arch": ARCH}, 'timeouts': {'action': {'minutes': 1}, 'job': {'minutes': 5}}, 'visibility': 'public'} if PASSWORD: boot = [action['boot'] for action in job_def['actions'] if 'boot' in action][0] boot['auto_login'].update({ "password_prompt": "Password:", "password": PASSWORD }) return job_def def main(): """ submit using XMLRPC """ image = os.path.realpath(IMAGE) url = "http://%s:%s@%s//RPC2" % (USERNAME, TOKEN, HOSTNAME) server = xmlrpclib.ServerProxy(url) job_id = server.scheduler.submit_job(yaml.dump(job(image))) print job_id return 0 if __name__ == '__main__': import sys sys.exit(main()) vmdebootstrap-1.4/examples/vmdebootstrap.txt0000644000175000017500000000232712420531347021424 0ustar neilneil00000000000000 # Run vmdebootstrap script to create image sudo \ vmdebootstrap \ --log ../log \ --log-level debug \ --size 3G \ --image $IMAGE.img \ --verbose \ --mirror $MIRROR \ --customize "$basedir/bin/freedombox-customize" \ --arch $ARCHITECTURE \ --distribution $SUITE \ $extra_opts \ $pkgopts Needs u-boot:armhf & linux-image-armmp:armhf in the image. sudo ./vmdebootstrap --image ../images/test.img \ --size 1g --owner $(whoami) --verbose \ --mirror http://mirror.bytemark.co.uk/debian \ --log ../images/test.log --log-level debug \ --arch armhf \ --foreign /usr/bin/qemu-arm-static \ --no-extlinux \ --no-kernel \ --package u-boot \ --package linux-image-armmp \ --distribution sid \ --bootsize 20m --boottype vfat # copy u-boot specific files # copy auto-serial-console to /bin/ sudo vmdebootstrap \ --enable-dhcp \ --serial-console --serial-console-command='/bin/auto-serial-console' \ --root-password='root' \ --verbose \ "$@" sudo vmdebootstrap \ --enable-dhcp --no-kernel --package=linux-image-generic \ --serial-console --serial-console-command='/bin/auto-serial-console' \ --root-password='root' --hostname='ubuntu' --user=user/pass --sudo \ --verbose \ vmdebootstrap-1.4/examples/beagleboneblack-customise.sh0000755000175000017500000000261712645254775023444 0ustar neilneil00000000000000#!/bin/sh set -e rootdir=$1 # copy u-boot to the boot partition cp $rootdir/usr/lib/u-boot/am335x_boneblack/MLO $rootdir/boot/MLO cp $rootdir/usr/lib/u-boot/am335x_boneblack/u-boot.img $rootdir/boot/u-boot.img # Setup uEnv.txt kernelVersion=$(basename `dirname $rootdir/usr/lib/*/am335x-boneblack.dtb`) version=$(echo $kernelVersion | sed 's/linux-image-\(.*\)/\1/') initRd=initrd.img-$version vmlinuz=vmlinuz-$version # uEnv.txt for Beaglebone # based on https://github.com/beagleboard/image-builder/blob/master/target/boot/beagleboard.org.txt cat >> $rootdir/boot/uEnv.txt < $rootdir/etc/modprobe.d/qemu-blacklist.conf vmdebootstrap-1.4/NEWS0000644000175000017500000000101212420531347014641 0ustar neilneil00000000000000NEWS for vmdebootstrap ====================== Version 0.3, unreleased ----------------------- * Add example customisation scripts to complement the support for other architectures which need work beyond the scope of available packages. Version 0.2, released 2013-11-15 -------------------------------- * Add support for building images for other architecture other than the native one. Patch by Petter Reinholdtsen. Version 0.1.0, released 2013-07-23 ---------------------------------- * First actual release. vmdebootstrap-1.4/yarns/0000755000175000017500000000000012650743704015313 5ustar neilneil00000000000000vmdebootstrap-1.4/yarns/Makefile0000644000175000017500000000213712617135440016751 0ustar neilneil00000000000000# Copyright 2015 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # =*= License: GPL-3+ =*= yarns = $(shell ls [0-9][0-9][0-9]-*.yarn) all: vmdebootstrap-yarns.pdf vmdebootstrap-yarns.html vmdebootstrap-yarns.pdf: $(yarns) Makefile pandoc --chapters \ --toc \ --number-sections \ -V documentclass:report \ -o $@ $(yarns) vmdebootstrap-yarns.html: $(yarns) Makefile yarns.css pandoc --chapters \ --toc \ --number-sections \ --standalone \ --self-contained \ -H yarns.css \ -o $@ $(yarns) vmdebootstrap-1.4/yarns/yarns.css0000644000175000017500000000173112617135440017156 0ustar neilneil00000000000000 vmdebootstrap-1.4/yarns/300-slow-build-tests.yarn0000644000175000017500000000430312650732660021726 0ustar neilneil00000000000000# Slow image building tests In this chapter, we have test scenarios that actually build an image and test the output. The images are not booted, but that may be added later. Instead, all the tests on the images are static. These tests are slow, since building images is slow. ## Build a very basic Debian 8 image SCENARIO build a basic Debian 8 image ASSUMING build tests are requested GIVEN user wants to build an image FOO.img that is 2GiB in size WHEN the user runs vmdebootstrap --sparse --extlinux THEN the image has the correct size AND the partition count of the image is 1 AND partition 1 has the boot flag set AND partition 1 has an ext4 filesystem AND partition 1 has file /etc/debian_version matching ^8\..*$ SCENARIO build a Debian 8 image with grub ASSUMING build tests are requested GIVEN user wants to build an image FOO.img that is 2GiB in size WHEN the user runs vmdebootstrap --grub THEN the image has the correct size AND the partition count of the image is 1 AND partition 1 has the boot flag set AND partition 1 has an ext4 filesystem AND partition 1 has file /boot/grub/grub.cfg matching ^### BEGIN /etc/grub.d/00_header ###$ AND partition 1 has file /etc/fstab matching ^\S+\s+\/\s+ext4\s+errors=remount-ro\s+\d\s+\d$ SCENARIO build a Debian 8 image with uefi ASSUMING build tests are requested GIVEN user wants to build an image FOO.img that is 2GiB in size WHEN the user runs vmdebootstrap --grub --use-uefi THEN the image has the correct size AND the partition count of the image is 2 AND partition 1 has an vfat filesystem AND partition 2 has file /boot/grub/grub.cfg matching ^### BEGIN /etc/grub.d/00_header ###$ SCENARIO build a Debian 8 image with btrfs and grub ASSUMING build tests are requested GIVEN user wants to build an image FOO.img that is 2GiB in size WHEN the user runs vmdebootstrap --roottype=btrfs --grub THEN the image has the correct size AND the partition count of the image is 1 AND partition 1 has the boot flag set AND partition 1 has an btrfs filesystem AND partition 1 has file /etc/fstab matching ^\S+\s+\/\s+btrfs\s+defaults\s+\d\s+\d$ vmdebootstrap-1.4/yarns/200-fast-tests.yarn0000644000175000017500000002101312650736105020574 0ustar neilneil00000000000000# Fast option check tests This chapter contains fast scenarios that test vmdebootstrap option handling. These scenarios do not actually build images, they only verify that vmdebootstrap parses the command line correctly. SCENARIO --squash and --image used together ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --squash=FOO --image=BAR --dry-run THEN vmdebootstrap exited with a non-zero exit code AND vmdebootstrap wrote an error message matching --squash AND vmdebootstrap wrote an error message matching --image SCENARIO --squash and --arch arm64 used together ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --squash=FOO --arch=arm64 --dry-run THEN vmdebootstrap exited with a zero exit code SCENARIO --use-uefi and --arch arm64 used together ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --image=FOO --grub --use-uefi --arch=arm64 --dry-run THEN vmdebootstrap exited with a zero exit code SCENARIO --arch arm64 used with --image but without --use-uefi ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --image=FOO --foreign=PATH --arch=arm64 --dry-run THEN vmdebootstrap exited with a non-zero exit code AND vmdebootstrap wrote an error message matching UEFI AND vmdebootstrap wrote an error message matching arm64 SCENARIO --use-uefi and --arch arm64 used without grub ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --image=FOO --use-uefi --arch=arm64 --dry-run THEN vmdebootstrap exited with a non-zero exit code AND vmdebootstrap wrote an error message matching UEFI AND vmdebootstrap wrote an error message matching Grub SCENARIO --use-uefi and --arch arm64 used without image ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --grub --use-uefi --arch=arm64 --dry-run THEN vmdebootstrap exited with a non-zero exit code AND vmdebootstrap wrote an error message matching disk image filename AND vmdebootstrap wrote an error message matching squash SCENARIO --image used with a zero size ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --image=FOO --size=0 --dry-run THEN vmdebootstrap exited with a non-zero exit code AND vmdebootstrap wrote an error message matching disk image AND vmdebootstrap wrote an error message matching must SCENARIO Debian distribution stable name check ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --image=FOO --distribution=stable --dry-run THEN vmdebootstrap exited with a zero exit code SCENARIO Debian distribution jessie name check ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --image=FOO --distribution=jessie --dry-run THEN vmdebootstrap exited with a zero exit code SCENARIO allow use of arch and foreign options ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --image=FOO --arch=armhf --foreign=PATH --dry-run THEN vmdebootstrap exited with a zero exit code SCENARIO disallow use of uefi on unsupported architectures ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --image=FOO --grub --use-uefi --arch=armel --dry-run THEN vmdebootstrap exited with a non-zero exit code AND vmdebootstrap wrote an error message matching armel AND vmdebootstrap wrote an error message matching not a supported AND vmdebootstrap wrote an error message matching UEFI SCENARIO disallow use of bootpartition if ESP already set ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --image=FOO --bootoffset=1024 --grub --use-uefi --dry-run THEN vmdebootstrap exited with a non-zero exit code AND vmdebootstrap wrote an error message matching separate boot AND vmdebootstrap wrote an error message matching not supported AND vmdebootstrap wrote an error message matching UEFI SCENARIO disallow UEFI support on wheezy except on amd64 ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --image=FOO --grub --use-uefi --distribution=wheezy --arch=arm64 --dry-run THEN vmdebootstrap exited with a non-zero exit code AND vmdebootstrap wrote an error message matching Only amd64 AND vmdebootstrap wrote an error message matching supports UEFI AND vmdebootstrap wrote an error message matching Wheezy SCENARIO disallow apt-mirror without configure-apt ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --image=FOO --apt-mirror=mirror THEN vmdebootstrap exited with a non-zero exit code AND vmdebootstrap wrote an error message matching requires --configure-apt as well SCENARIO default includes the kernel package ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --image=FOO --arch=amd64 --dry-run THEN vmdebootstrap exited with a zero exit code AND vmdebootstrap wrote a message matching linux-image-amd64 SCENARIO no-linux omits the kernel package ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --no-kernel --image=FOO --arch=amd64 --dry-run THEN vmdebootstrap exited with a zero exit code AND vmdebootstrap wrote a message not matching linux-image-amd64 SCENARIO kernel-package includes the kernel package ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --kernel-package unlikely --image=FOO --arch=amd64 --dry-run THEN vmdebootstrap exited with a zero exit code AND vmdebootstrap wrote a message not matching linux-image-amd64 AND vmdebootstrap wrote a message matching unlikely SCENARIO no-kernel overrides kernel-package ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --no-kernel --kernel-package unlikely --image=FOO --arch=amd64 --dry-run THEN vmdebootstrap exited with a zero exit code AND vmdebootstrap wrote a message not matching linux-image-amd64 AND vmdebootstrap wrote a message not matching unlikely SCENARIO package includes the specified package ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --package unlikely --image=FOO --arch=amd64 --dry-run THEN vmdebootstrap exited with a zero exit code AND vmdebootstrap wrote a message matching linux-image-amd64 AND vmdebootstrap wrote a message matching unlikely SCENARIO only images can be converted to qcow2 ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --convert-qcow2 --tarball --arch=amd64 --dry-run THEN vmdebootstrap exited with a non-zero exit code AND vmdebootstrap wrote an error message matching qcow2 can only AND vmdebootstrap wrote an error message matching with --image SCENARIO tarball is not usable with image ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --image=FOO --tarball=FOO --arch=amd64 --dry-run THEN vmdebootstrap exited with a non-zero exit code AND vmdebootstrap wrote an error message matching not both SCENARIO tarball is not usable with squash ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --squash=FOO --tarball=FOO --arch=amd64 --dry-run THEN vmdebootstrap exited with a non-zero exit code AND vmdebootstrap wrote an error message matching not both SCENARIO masking systemd-networkd without updating initramfs ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --image=FOO --no-systemd-networkd --no-update-initramfs --dry-run THEN vmdebootstrap exited with a non-zero exit code AND vmdebootstrap wrote an error message matching requires updating the SCENARIO btrfs not yet supported with extlinux ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --image=FOO --roottype btrfs --extlinux --dry-run THEN vmdebootstrap exited with a non-zero exit code AND vmdebootstrap wrote an error message matching not yet supported SCENARIO btrfs not yet supported without disabling extlinux ASSUMING fast tests are requested WHEN user attempts to run vmdebootstrap ... --image=FOO --roottype btrfs --dry-run THEN vmdebootstrap exited with a non-zero exit code AND vmdebootstrap wrote an error message matching not yet supported vmdebootstrap-1.4/yarns/000-meta.yarn0000644000175000017500000000010412617135440017417 0ustar neilneil00000000000000--- title: vmdebootstrap integration test suite date: PRE-ALPHA ... vmdebootstrap-1.4/yarns/900-implements.yarn0000644000175000017500000001031512646720751020673 0ustar neilneil00000000000000# Scenario step implementations This chapter has implementations of scenario steps used elsewhere in this document. ## Should a scenario be run? We provide a way to control which classes of scenarios get run. This is done by passing in an environment variable `TESTS` to vmdebootstrap (`--env` option), with a comma-separated list of classes: * `fast`---run fast tests * `build`---run scenarios that do builds If `TESTS` is not set, everything gets run. Scenarios can use the ASSUMING statements defined here to let the user to allow them to run or not to run. IMPLEMENTS ASSUMING (fast|build) tests are requested support_required test_requested "$MATCH_1" if `MIRROR` is set, use the default http mirror IMPLEMENTS ASSUMING (local|http) tests are requested support_required build_support_required mirror_requested "$MATCH_1" ## Building an image, or attempting to To keep individual steps shorter, we provide some steps to set common parts, such as the name of the image being built. IMPLEMENTS GIVEN user wants to build an image (\S+) that is (\S+) in size remember_setting IMAGE "$MATCH_1" remember_setting IMAGE_SIZE "$(size_in_bytes "$MATCH_2")" Actually build an image. This looks like it can invoke any command, but it's actually restricted to vmdebootstrap in the source tree. IMPLEMENTS WHEN the user runs vmdebootstrap (.*) PYTHONPATH="$SRCDIR" "$SRCDIR/bin/vmdebootstrap" \ --image "$IMAGE" \ --mirror ${MIRROR:=http://httpredir.debian.org/debian/} \ --size "$IMAGE_SIZE" \ $MATCH_1 Attempt to run vmdebootstrap, but allow it it to fail. Capture exit code and stderr. IMPLEMENTS WHEN user attempts to run vmdebootstrap (.*) if PYTHONPATH="$SRCDIR" "$SRCDIR/bin/vmdebootstrap" \ $MATCH_1 2> vmdebootstrap.stderr 1>vmdebootstrap.stdout then echo 0 > vmdebootstrap.exit else echo $? > vmdebootstrap.exit fi Check for whether the attempted vmdebootstrap run had the desired exit code. IMPLEMENTS THEN vmdebootstrap exited with a non-zero exit code grep -vFx 0 vmdebootstrap.exit IMPLEMENTS THEN vmdebootstrap exited with a zero exit code grep -Fx 0 vmdebootstrap.exit Check the stderr of the attempted vmdebootstrap run. IMPLEMENTS THEN vmdebootstrap wrote an error message matching (.+) grep -P -e "$MATCH_1" vmdebootstrap.stderr Check the stdout of the dry-run vmdebootstrap run. IMPLEMENTS THEN vmdebootstrap wrote a message matching (.+) grep -P -e "$MATCH_1" vmdebootstrap.stdout Exclude a pattern from the stdout of the dry-run vmdebootstrap run. IMPLEMENTS THEN vmdebootstrap wrote a message not matching (.+) grep -P -v -e "$MATCH_1" vmdebootstrap.stdout ## Static tests on disk images The steps in this section do static tests of disk image. These all operate on the image specified in the step "GIVEN user wants to build...". Test the size of an image. This tests the length, not disk usage, of the image. IMPLEMENTS THEN the image has the correct size actual="$(stat -c %s "$IMAGE")" [ "$actual" = "$IMAGE_SIZE" ] Check the partition table on the image. IMPLEMENTS THEN the partition count of the image is (\d+) parted --script "$IMAGE" print | sed '1,/^Number/d' | grep -c . | grep -Fx $MATCH_1 Check partition boot flag. IMPLEMENTS THEN partition (\d+) has the boot flag set parted --script "$IMAGE" print | awk -v "PART=$MATCH_1" '/^ [0-9]+ / && $1 == PART && $7 == "boot"' | grep . Check filesystem on a partition. This checks the actual filesystem, not a type declared in the partition table. IMPLEMENTS THEN partition (\d+) has an? (\S+) filesystem device="$(kpartx_image_partition "$IMAGE" "$MATCH_1")" trap "unkpartx_image \"$IMAGE\"" EXIT blkid "$device" | grep "TYPE=\"$MATCH_2\"" Check that the partition contains a file with some content matching a regular expression. IMPLEMENTS THEN partition (\d+) has file (\S+) matching (.+) device="$(kpartx_image_partition "$IMAGE" "$MATCH_1")" trap "unkpartx_image \"$IMAGE\"" EXIT mp="$(mktemp -d)" mount -r "$device" "$mp" trap "umount \"$mp\"; unkpartx_image \"$IMAGE\"" EXIT grep -P -e "$MATCH_3" "$mp/$MATCH_2" vmdebootstrap-1.4/yarns/800-future.yarn0000644000175000017500000000106112635272114020016 0ustar neilneil00000000000000# Thoughts for future tests * Build image, run it under Qemu or some other way. Then do some basic testing of the running system. For example, boot it under qemu-system-i386, with a serial consoler, and verify that it's possible to log in via the serial consoler, and ping the host. More detailed testing of built systems may or may not be appropriate: a lot of more intricate testing may be beyond the scope of testing of vmdebootstrap. * Use the lava-submit.py script to submit a local file to LAVA using the locally installed lava-dispatcher. vmdebootstrap-1.4/yarns/100-intro.yarn0000644000175000017500000000071312617135440017633 0ustar neilneil00000000000000# Introduction This document is the integration test suite for vmdebootstrap. It is both human-readable and machine-executable. It is written for the [yarn][] tool. This document is aimed at those developing vmdebootstrap. * See the vmdebootstrap documentation about what the software is for and how it is used. * See the [yarn documentation][] for how to use yarn. [yarn]: http://liw.fi/cmdtest/ [yarn documentation]: http://liw.fi/cmdtest/README.yarn/ vmdebootstrap-1.4/yarns/shell.lib0000644000175000017500000000520612646720757017125 0ustar neilneil00000000000000# A shell library for yarn scenario step implementations. # Copyright 2015 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # =*= License: GPL-3+ =*= # Is running of scenarios of a given type requested? If nothing is # requested ($TESTS is empty), then everything is. test_requested() { if [ "${TESTS:-}" = "" ] then return 0 elif echo "${TESTS}" | tr , '\n' | grep -Fx "$1" then return 0 else return 1 fi } support_required() { if [ ! -x /usr/bin/cmdtest ]; then echo "All tests need the cmdtest package." return 1 fi } build_support_required() { if [ ! -x /usr/bin/bc ]; then echo "Build tests need the bc package." return 1 fi } # Convert a size with a unit, such a kB, to plain bytes. size_in_bytes() { local amount="$(echo "$1" | sed 's/[^0-9]*$//')" local unit="$(echo "$1" | sed 's/^[0-9]*//' | tr A-Z a-z)" local factor=1 case "$unit" in k|kb) factor=1000 ;; m|mb) factor=1000000 ;; g|gb) factor=1000000000 ;; t|tb) factor=1000000000000 ;; ki|kib) factor=1024 ;; mi|mib) factor=1048576 ;; gi|gib) factor=1073741824 ;; ti|tib) factor=1099511627776 ;; esac echo "$amount * $factor" | bc -lq } # Make a partition in a disk image accessible as a block device. # Return it's device file. kpartx_image_partition() { kpartx -sav "$1" | awk -v "n=$2" 'NR == n { print "/dev/mapper/" $3 }' } # Undo kpartx_image_partition. unkpartx_image() { kpartx -d "$1" } # Rembember and retrieve settings between scenario steps. This is # implemented by a shell script snippet that gets sourced at the end # of this shell library. remember_setting() { echo "$1=$2" >> "$DATADIR/settings.sh" } if [ -e "$DATADIR/settings.sh" ] then . "$DATADIR/settings.sh" fi # Make sure the current working directory is $DATADIR, not $SRCDIR. # This makes it a little simpler to write scenario step # implementations by not having to be specifying $DATADIR explicitly # everywhere. cd "$DATADIR" vmdebootstrap-1.4/yarns/run-tests0000755000175000017500000000156212617135440017204 0ustar neilneil00000000000000#!/bin/sh # Copyright 2015 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # =*= License: GPL-3+ =*= set -eu die() { echo "$@" 1>&2 exit 1 } [ -e bin/vmdebootstrap ] || die "You need to run this at root of source tree" yarn -s yarns/shell.lib yarns/*.yarn "$@" vmdebootstrap-1.4/man/0000755000175000017500000000000012650743704014732 5ustar neilneil00000000000000vmdebootstrap-1.4/man/conf.py0000644000175000017500000001555612646263614016247 0ustar neilneil00000000000000# -*- coding: utf-8 -*- # # vmdebootstrap documentation build configuration file # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys import os import subprocess # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.append(os.path.abspath('..')) # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage'] # Configuration for sphinx.ext.todo todo_include_todos = True # Add any paths that contain templates here, relative to this directory. templates_path = [] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. # source_encoding = 'utf-8' # The master toctree document. master_doc = 'index' # General information about the project. project = u'vmdebootstrap' copyright = u'2015, Neil Williams' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = subprocess.Popen(['python', 'setup.py', '-V'], cwd=r'..', stdout=subprocess.PIPE).stdout.read() # The full version, including alpha/beta/rc tags. release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # today = '' # Else, today_fmt is used as the format for a strftime call. # today_fmt = '%B %d, %Y' # List of documents that shouldn't be included in the build. # unused_docs = [] # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = [] # The reST default role (used for this markup: `text`) to use for all documents. # default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. # add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). # add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. # show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". # html_title = None # A shorter title for the navigation bar. Default is the same as html_title. # html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. # html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. # html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. # html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. # html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. # html_additional_pages = {} # If false, no module index is generated. # html_use_modindex = True # If false, no index is generated. # html_use_index = True # If true, the index is split into individual pages for each letter. # html_split_index = False # If true, links to the reST sources are added to the pages. # html_show_sourcelink = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. # html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). # html_file_suffix = '' # Output file base name for HTML help builder. htmlhelp_basename = 'vmdebootstrap' # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). # latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). # latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'vmdebootstrap.tex', u'VMDebootstrap Documentation', u'Neil Williams', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. # latex_use_parts = False # Additional stuff for the LaTeX preamble. # latex_preamble = '' # Documents to append as an appendix to all manuals. # latex_appendices = [] # If false, no module index is generated. # latex_use_modindex = True # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'http://docs.python.org/': None} # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('overview', 'vmdebootstrap', u'install basic Debian system into virtual disk image', [u'Neil Williams'], 8), ] vmdebootstrap-1.4/man/Makefile0000644000175000017500000001273412617135212016371 0ustar neilneil00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: -rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/LavaDispatcher.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/LavaDispatcher.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/LavaDispatcher" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/LavaDispatcher" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." vmdebootstrap-1.4/man/index.rst0000644000175000017500000000006012617135440016562 0ustar neilneil00000000000000.. toctree:: :maxdepth: 2 overview.rst vmdebootstrap-1.4/man/overview.rst0000777000175000017500000000000012617135440022702 2../doc/overview.rstustar neilneil00000000000000